Verifies whether objects/variable are equal to an expected value Class: org.apache.hadoop.hive.accumulo.TestAccumuloHiveRow InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.accumulo.TestAccumuloStorageHandler BooleanVerifier EqualityVerifier HybridVerifier jobProperties=new HashMap();
props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,"cf:cq1,cf:cq2,cf:cq3");
props.setProperty(AccumuloSerDeParameters.TABLE_NAME,"table");
props.setProperty(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY,"foo");
Mockito.when(tableDesc.getProperties()).thenReturn(props);
storageHandler.configureOutputJobProperties(tableDesc,jobProperties);
Assert.assertEquals(3,jobProperties.size());
Assert.assertTrue("Job properties did not contain column mappings",jobProperties.containsKey(AccumuloSerDeParameters.COLUMN_MAPPINGS));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS),jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS));
Assert.assertTrue("Job properties did not contain accumulo table name",jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME),jobProperties.get(AccumuloSerDeParameters.TABLE_NAME));
Assert.assertTrue("Job properties did not contain visibility label",jobProperties.containsKey(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY),jobProperties.get(AccumuloSerDeParameters.VISIBILITY_LABEL_KEY));
}
BooleanVerifier EqualityVerifier HybridVerifier jobProperties=new HashMap();
props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,"cf:cq1,cf:cq2,cf:cq3");
props.setProperty(AccumuloSerDeParameters.TABLE_NAME,"table");
props.setProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY,"true");
props.setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE,ColumnEncoding.BINARY.getName());
props.setProperty(AccumuloSerDeParameters.AUTHORIZATIONS_KEY,"foo,bar");
Mockito.when(tableDesc.getProperties()).thenReturn(props);
storageHandler.configureInputJobProperties(tableDesc,jobProperties);
Assert.assertEquals(5,jobProperties.size());
Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.COLUMN_MAPPINGS));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS),jobProperties.get(AccumuloSerDeParameters.COLUMN_MAPPINGS));
Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.TABLE_NAME));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.TABLE_NAME),jobProperties.get(AccumuloSerDeParameters.TABLE_NAME));
Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY),jobProperties.get(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY));
Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),jobProperties.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE));
Assert.assertTrue(jobProperties.containsKey(AccumuloSerDeParameters.AUTHORIZATIONS_KEY));
Assert.assertEquals(props.getProperty(AccumuloSerDeParameters.AUTHORIZATIONS_KEY),jobProperties.get(AccumuloSerDeParameters.AUTHORIZATIONS_KEY));
}
Class: org.apache.hadoop.hive.accumulo.TestHiveAccumuloHelper InternalCallVerifier EqualityVerifier > tokens=new ArrayList<>();
Text service=new Text("service");
Token token=Mockito.mock(Token.class);
tokens.add(token);
Mockito.when(ugi.getTokens()).thenReturn(tokens);
Mockito.when(token.getKind()).thenReturn(HiveAccumuloHelper.ACCUMULO_SERVICE);
Mockito.when(token.getService()).thenReturn(service);
try {
helper.addTokenFromUserToJobConf(ugi,jobConf);
}
catch ( IOException e) {
log.info("Ignoring exception, likely coming from Hadoop 1",e);
return;
}
Collection> credTokens=jobConf.getCredentials().getAllTokens();
assertEquals(1,credTokens.size());
assertEquals(service,credTokens.iterator().next().getService());
}
InternalCallVerifier EqualityVerifier > tokens=jobConf.getCredentials().getAllTokens();
assertEquals(1,tokens.size());
assertEquals(service,tokens.iterator().next().getService());
}
Class: org.apache.hadoop.hive.accumulo.TestLazyAccumuloMap APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ").get(0),new byte[]{(byte)1,(byte)2},0,nullSequence,false,(byte)0);
LazyAccumuloMap map=new LazyAccumuloMap((LazyMapObjectInspector)oi);
map.init(row,mapping);
Assert.assertEquals(3,map.getMapSize());
Object o=map.getMapValueElement(new IntWritable(1));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(2),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(2));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(4),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(3));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(6),((LazyInteger)o).getWritableObject());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ").get(0),new byte[]{(byte)1,(byte)2},0,nullSequence,false,(byte)0);
LazyAccumuloMap map=new LazyAccumuloMap((LazyMapObjectInspector)oi);
map.init(row,mapping);
Assert.assertEquals(2,map.getMapSize());
Object o=map.getMapValueElement(new Text("foo"));
Assert.assertNotNull(o);
Assert.assertEquals(new Text("bar"),((LazyString)o).getWritableObject());
o=map.getMapValueElement(new Text("bar"));
Assert.assertNotNull(o);
Assert.assertEquals(new Text("foo"),((LazyString)o).getWritableObject());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ").get(0),new byte[]{(byte)1,(byte)2},0,nullSequence,false,(byte)0);
LazyAccumuloMap map=new LazyAccumuloMap((LazyMapObjectInspector)oi);
map.init(row,mapping);
Assert.assertEquals(3,map.getMapSize());
Object o=map.getMapValueElement(new IntWritable(1));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(2),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(2));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(4),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(3));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(6),((LazyInteger)o).getWritableObject());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ").get(0),new byte[]{(byte)1,(byte)2},0,nullSequence,false,(byte)0);
LazyAccumuloMap map=new LazyAccumuloMap((LazyMapObjectInspector)oi);
map.init(row,mapping);
Assert.assertEquals(3,map.getMapSize());
Object o=map.getMapValueElement(new IntWritable(1));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(2),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(2));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(4),((LazyInteger)o).getWritableObject());
o=map.getMapValueElement(new IntWritable(3));
Assert.assertNotNull(o);
Assert.assertEquals(new IntWritable(6),((LazyInteger)o).getWritableObject());
}
Class: org.apache.hadoop.hive.accumulo.TestLazyAccumuloRow APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columns=Arrays.asList("row","given_name","surname","age","weight","height");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo);
LazySimpleStructObjectInspector objectInspector=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(columns,types,LazySerDeParameters.DefaultSeparators,new Text("\\N"),false,false,(byte)'\\');
DefaultAccumuloRowIdFactory rowIdFactory=new DefaultAccumuloRowIdFactory();
Properties props=new Properties();
props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid#s,personal:given_name#s,personal:surname#s,personal:age,personal:weight,personal:height");
props.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
props.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
props.setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE,ColumnEncoding.BINARY.getName());
AccumuloSerDeParameters params=new AccumuloSerDeParameters(new Configuration(),props,AccumuloSerDe.class.getName());
rowIdFactory.init(params,props);
ByteArrayOutputStream baos=new ByteArrayOutputStream();
DataOutputStream out=new DataOutputStream(baos);
LazyAccumuloRow lazyRow=new LazyAccumuloRow(objectInspector);
AccumuloHiveRow hiveRow=new AccumuloHiveRow("1");
hiveRow.add("personal","given_name","Bob".getBytes());
hiveRow.add("personal","surname","Stevens".getBytes());
out.writeInt(30);
hiveRow.add("personal","age",baos.toByteArray());
baos.reset();
out.writeInt(200);
hiveRow.add("personal","weight",baos.toByteArray());
baos.reset();
out.writeInt(72);
hiveRow.add("personal","height",baos.toByteArray());
ColumnMapper columnMapper=params.getColumnMapper();
lazyRow.init(hiveRow,columnMapper.getColumnMappings(),rowIdFactory);
Object o=lazyRow.getField(0);
Assert.assertNotNull(o);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("1",((LazyString)o).toString());
o=lazyRow.getField(1);
Assert.assertNotNull(o);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("Bob",((LazyString)o).toString());
o=lazyRow.getField(2);
Assert.assertNotNull(o);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("Stevens",((LazyString)o).toString());
o=lazyRow.getField(3);
Assert.assertNotNull(o);
Assert.assertEquals(LazyDioInteger.class,o.getClass());
Assert.assertEquals("30",((LazyDioInteger)o).toString());
o=lazyRow.getField(4);
Assert.assertNotNull(o);
Assert.assertEquals(LazyDioInteger.class,o.getClass());
Assert.assertEquals("200",((LazyDioInteger)o).toString());
o=lazyRow.getField(5);
Assert.assertNotNull(o);
Assert.assertEquals(LazyDioInteger.class,o.getClass());
Assert.assertEquals("72",((LazyDioInteger)o).toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","given_name","surname","age","weight","height");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo);
LazySimpleStructObjectInspector objectInspector=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(columns,types,LazySerDeParameters.DefaultSeparators,new Text("\\N"),false,false,(byte)'\\');
DefaultAccumuloRowIdFactory rowIdFactory=new DefaultAccumuloRowIdFactory();
Properties props=new Properties();
props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,personal:given_name,personal:surname,personal:age,personal:weight,personal:height");
props.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
props.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters params=new AccumuloSerDeParameters(new Configuration(),props,AccumuloSerDe.class.getName());
rowIdFactory.init(params,props);
LazyAccumuloRow lazyRow=new LazyAccumuloRow(objectInspector);
AccumuloHiveRow hiveRow=new AccumuloHiveRow("1");
hiveRow.add("personal","given_name","Bob".getBytes());
hiveRow.add("personal","surname","Stevens".getBytes());
hiveRow.add("personal","age","30".getBytes());
hiveRow.add("personal","weight","200".getBytes());
hiveRow.add("personal","height","72".getBytes());
ColumnMapper columnMapper=params.getColumnMapper();
lazyRow.init(hiveRow,columnMapper.getColumnMappings(),rowIdFactory);
Object o=lazyRow.getField(0);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("1",((LazyString)o).toString());
o=lazyRow.getField(1);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("Bob",((LazyString)o).toString());
o=lazyRow.getField(2);
Assert.assertEquals(LazyString.class,o.getClass());
Assert.assertEquals("Stevens",((LazyString)o).toString());
o=lazyRow.getField(3);
Assert.assertEquals(LazyInteger.class,o.getClass());
Assert.assertEquals("30",((LazyInteger)o).toString());
o=lazyRow.getField(4);
Assert.assertEquals(LazyInteger.class,o.getClass());
Assert.assertEquals("200",((LazyInteger)o).toString());
o=lazyRow.getField(5);
Assert.assertEquals(LazyInteger.class,o.getClass());
Assert.assertEquals("72",((LazyInteger)o).toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","1","2","3");
List types=Arrays.asList(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME),TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME));
LazySimpleStructObjectInspector objectInspector=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(columns,types,LazySerDeParameters.DefaultSeparators,new Text("\\N"),false,false,(byte)'\\');
DefaultAccumuloRowIdFactory rowIdFactory=new DefaultAccumuloRowIdFactory();
Properties props=new Properties();
props.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,cf:cq1,cf:cq2,cf:cq3");
props.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
props.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters params=new AccumuloSerDeParameters(new Configuration(),props,AccumuloSerDe.class.getName());
rowIdFactory.init(params,props);
ColumnMapper columnMapper=params.getColumnMapper();
LazyAccumuloRow lazyRow=new LazyAccumuloRow(objectInspector);
AccumuloHiveRow hiveRow=new AccumuloHiveRow("1");
hiveRow.add("cf","cq1","foo".getBytes());
hiveRow.add("cf","cq3","bar".getBytes());
lazyRow.init(hiveRow,columnMapper.getColumnMappings(),rowIdFactory);
Assert.assertEquals("{'row':'1','1':'foo','2':null,'3':'bar'}".replace('\'','"'),SerDeUtils.getJSONString(lazyRow,objectInspector));
Assert.assertEquals("{'row':'1','1':'foo','2':null,'3':'bar'}".replace('\'','"'),SerDeUtils.getJSONString(lazyRow,objectInspector));
}
Class: org.apache.hadoop.hive.accumulo.columns.TestColumnEncoding EqualityVerifier EqualityVerifier stringString=Maps.immutableEntry(ColumnEncoding.STRING,ColumnEncoding.STRING), stringBinary=Maps.immutableEntry(ColumnEncoding.STRING,ColumnEncoding.BINARY), binaryBinary=Maps.immutableEntry(ColumnEncoding.BINARY,ColumnEncoding.BINARY), binaryString=Maps.immutableEntry(ColumnEncoding.BINARY,ColumnEncoding.STRING);
Assert.assertEquals(stringString,ColumnEncoding.getMapEncoding("s:s"));
Assert.assertEquals(stringString,ColumnEncoding.getMapEncoding("s:string"));
Assert.assertEquals(stringString,ColumnEncoding.getMapEncoding("string:s"));
Assert.assertEquals(stringString,ColumnEncoding.getMapEncoding("string:string"));
Assert.assertEquals(stringBinary,ColumnEncoding.getMapEncoding("s:b"));
Assert.assertEquals(stringBinary,ColumnEncoding.getMapEncoding("string:b"));
Assert.assertEquals(stringBinary,ColumnEncoding.getMapEncoding("s:binary"));
Assert.assertEquals(stringBinary,ColumnEncoding.getMapEncoding("string:binary"));
Assert.assertEquals(binaryString,ColumnEncoding.getMapEncoding("b:s"));
Assert.assertEquals(binaryString,ColumnEncoding.getMapEncoding("b:string"));
Assert.assertEquals(binaryString,ColumnEncoding.getMapEncoding("binary:s"));
Assert.assertEquals(binaryString,ColumnEncoding.getMapEncoding("binary:string"));
Assert.assertEquals(binaryBinary,ColumnEncoding.getMapEncoding("b:b"));
Assert.assertEquals(binaryBinary,ColumnEncoding.getMapEncoding("binary:b"));
Assert.assertEquals(binaryBinary,ColumnEncoding.getMapEncoding("b:binary"));
Assert.assertEquals(binaryBinary,ColumnEncoding.getMapEncoding("binary:binary"));
}
EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.accumulo.columns.TestColumnMapper APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier rawMappings=Arrays.asList(AccumuloHiveConstants.ROWID,"cf:cq","cf:_","cf:qual");
List columnNames=Arrays.asList("row","col1","col2","col3");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
ColumnMapper mapper=new ColumnMapper(Joiner.on(AccumuloHiveConstants.COMMA).join(rawMappings),ColumnEncoding.STRING.getName(),columnNames,columnTypes);
List mappings=mapper.getColumnMappings();
Assert.assertEquals(rawMappings.size(),mappings.size());
Assert.assertEquals(mappings.size(),mapper.size());
Iterator rawIter=rawMappings.iterator();
Iterator iter=mappings.iterator();
for (int i=0; i < mappings.size() && iter.hasNext(); i++) {
String rawMapping=rawIter.next();
ColumnMapping mapping=iter.next();
ColumnMapping mappingByOffset=mapper.get(i);
Assert.assertEquals(mapping,mappingByOffset);
if (AccumuloHiveConstants.ROWID.equals(rawMapping)) {
Assert.assertEquals(HiveAccumuloRowIdColumnMapping.class,mapping.getClass());
}
else {
Assert.assertEquals(HiveAccumuloColumnMapping.class,mapping.getClass());
}
}
Assert.assertEquals(0,mapper.getRowIdOffset());
Assert.assertTrue(mapper.hasRowIdMapping());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier hiveColumns=Arrays.asList("rowid","col1","col2","col3");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo),TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo),TypeInfoFactory.stringTypeInfo);
List rawMappings=Arrays.asList(AccumuloHiveConstants.ROWID,"cf1:*","cf2:2*","cq3:bar\\*");
ColumnMapper mapper=new ColumnMapper(Joiner.on(AccumuloHiveConstants.COMMA).join(rawMappings),ColumnEncoding.BINARY.getName(),hiveColumns,columnTypes);
List mappings=mapper.getColumnMappings();
Assert.assertEquals(4,mappings.size());
Assert.assertEquals(HiveAccumuloRowIdColumnMapping.class,mappings.get(0).getClass());
Assert.assertEquals(HiveAccumuloMapColumnMapping.class,mappings.get(1).getClass());
Assert.assertEquals(HiveAccumuloMapColumnMapping.class,mappings.get(2).getClass());
Assert.assertEquals(HiveAccumuloColumnMapping.class,mappings.get(3).getClass());
HiveAccumuloRowIdColumnMapping row=(HiveAccumuloRowIdColumnMapping)mappings.get(0);
Assert.assertEquals(ColumnEncoding.BINARY,row.getEncoding());
Assert.assertEquals(hiveColumns.get(0),row.getColumnName());
Assert.assertEquals(columnTypes.get(0).toString(),row.getColumnType());
HiveAccumuloMapColumnMapping map=(HiveAccumuloMapColumnMapping)mappings.get(1);
Assert.assertEquals("cf1",map.getColumnFamily());
Assert.assertEquals("",map.getColumnQualifierPrefix());
Assert.assertEquals(ColumnEncoding.BINARY,map.getEncoding());
Assert.assertEquals(hiveColumns.get(1),map.getColumnName());
Assert.assertEquals(columnTypes.get(1).toString(),map.getColumnType());
map=(HiveAccumuloMapColumnMapping)mappings.get(2);
Assert.assertEquals("cf2",map.getColumnFamily());
Assert.assertEquals("2",map.getColumnQualifierPrefix());
Assert.assertEquals(ColumnEncoding.BINARY,map.getEncoding());
Assert.assertEquals(hiveColumns.get(2),map.getColumnName());
Assert.assertEquals(columnTypes.get(2).toString(),map.getColumnType());
HiveAccumuloColumnMapping column=(HiveAccumuloColumnMapping)mappings.get(3);
Assert.assertEquals("cq3",column.getColumnFamily());
Assert.assertEquals("bar*",column.getColumnQualifier());
Assert.assertEquals(ColumnEncoding.BINARY,column.getEncoding());
Assert.assertEquals(hiveColumns.get(3),column.getColumnName());
Assert.assertEquals(columnTypes.get(3).toString(),column.getColumnType());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier hiveColumns=Arrays.asList("rowid","col1","col2","col3");
List rawMappings=Arrays.asList(AccumuloHiveConstants.ROWID,"cf:cq","cf:_","cf:qual");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
ColumnMapper mapper=new ColumnMapper(Joiner.on(AccumuloHiveConstants.COMMA).join(rawMappings),null,hiveColumns,columnTypes);
String typeString=mapper.getTypesString();
String[] types=StringUtils.split(typeString,AccumuloHiveConstants.COLON);
Assert.assertEquals(rawMappings.size(),types.length);
for ( String type : types) {
Assert.assertEquals(serdeConstants.STRING_TYPE_NAME,type);
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier hiveColumns=Arrays.asList("rowid","col1","col2","col3");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
List rawMappings=Arrays.asList(AccumuloHiveConstants.ROWID,"cf:cq","cf:_","cf:qual");
ColumnMapper mapper=new ColumnMapper(Joiner.on(AccumuloHiveConstants.COMMA).join(rawMappings),null,hiveColumns,columnTypes);
for (int i=0; i < hiveColumns.size(); i++) {
String hiveColumn=hiveColumns.get(i), accumuloMapping=rawMappings.get(i);
ColumnMapping mapping=mapper.getColumnMappingForHiveColumn(hiveColumns,hiveColumn);
Assert.assertEquals(accumuloMapping,mapping.getMappingSpec());
}
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier hiveColumns=Arrays.asList("rowid","col1","col2","col3","col4");
List rawMappings=Arrays.asList(AccumuloHiveConstants.ROWID,"cf:cq","cf:_#s","cf:qual#s","cf:qual2");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
ColumnMapper mapper=new ColumnMapper(Joiner.on(AccumuloHiveConstants.COMMA).join(rawMappings),ColumnEncoding.BINARY.getName(),hiveColumns,columnTypes);
List mappings=mapper.getColumnMappings();
Assert.assertEquals(5,mappings.size());
Assert.assertEquals(ColumnEncoding.BINARY,mappings.get(0).getEncoding());
Assert.assertEquals(columnTypes.get(0).toString(),mappings.get(0).getColumnType());
Assert.assertEquals(ColumnEncoding.BINARY,mappings.get(1).getEncoding());
Assert.assertEquals(columnTypes.get(1).toString(),mappings.get(1).getColumnType());
Assert.assertEquals(ColumnEncoding.STRING,mappings.get(2).getEncoding());
Assert.assertEquals(columnTypes.get(2).toString(),mappings.get(2).getColumnType());
Assert.assertEquals(ColumnEncoding.STRING,mappings.get(3).getEncoding());
Assert.assertEquals(columnTypes.get(3).toString(),mappings.get(3).getColumnType());
Assert.assertEquals(ColumnEncoding.BINARY,mappings.get(4).getEncoding());
Assert.assertEquals(columnTypes.get(4).toString(),mappings.get(4).getColumnType());
}
Class: org.apache.hadoop.hive.accumulo.columns.TestColumnMappingFactory APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier pair=ColumnMappingFactory.parseMapping(cf + ":" + cq);
Assert.assertEquals("c:f",pair.getKey());
Assert.assertEquals(cq,pair.getValue());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier pair=ColumnMappingFactory.parseMapping(cf + ":" + cq);
Assert.assertEquals(cf,pair.getKey());
Assert.assertEquals(cq,pair.getValue());
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier pair=ColumnMappingFactory.parseMapping(cf + ":" + cq);
Assert.assertEquals("c:f",pair.getKey());
Assert.assertEquals("cq1:cq2",pair.getValue());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.accumulo.columns.TestHiveAccumuloColumnMapping InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier reader=inputformat.getRecordReader(splits[0],conf,null);
Text rowId=new Text("r1");
AccumuloHiveRow row=new AccumuloHiveRow();
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"brian".getBytes());
rowId=new Text("r2");
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"mark".getBytes());
rowId=new Text("r3");
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"dennis".getBytes());
rowId=new Text("r4");
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"frank".getBytes());
assertFalse(reader.next(rowId,row));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo)));
Set> pairs=inputformat.getPairCollection(columnMapper.getColumnMappings());
Assert.assertEquals(1,pairs.size());
Pair cfCq=pairs.iterator().next();
Assert.assertEquals("cf",cfCq.getFirst().toString());
Assert.assertNull(cfCq.getSecond());
}
BranchVerifier BooleanVerifier EqualityVerifier HybridVerifier kv : scan) {
SortedMap items=PrimitiveComparisonFilter.decodeRow(kv.getKey(),kv.getValue());
for ( Map.Entry item : items.entrySet()) {
assertEquals(item.getKey().getRow().toString(),"r1");
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName=true;
assertArrayEquals(item.getValue().get(),"brian".getBytes());
}
else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid=true;
assertArrayEquals(item.getValue().get(),parseIntBytes("1"));
}
else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees=true;
assertArrayEquals(item.getValue().get(),parseDoubleBytes("44.5"));
}
else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis=true;
assertArrayEquals(item.getValue().get(),parseLongBytes("555"));
}
}
}
assertTrue(foundDegrees & foundMillis & foundName& foundSid);
}
BooleanVerifier EqualityVerifier HybridVerifier kv : scan) {
boolean foundName=false;
boolean foundSid=false;
boolean foundDegrees=false;
boolean foundMillis=false;
SortedMap items=PrimitiveComparisonFilter.decodeRow(kv.getKey(),kv.getValue());
for ( Map.Entry item : items.entrySet()) {
if (item.getKey().getRow().toString().equals("r2")) {
foundMark=true;
}
else if (item.getKey().getRow().toString().equals("r3")) {
foundDennis=true;
}
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName=true;
}
else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid=true;
}
else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees=true;
}
else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis=true;
}
}
totalCount++;
assertTrue(foundDegrees & foundMillis & foundName& foundSid);
}
assertTrue(foundDennis & foundMark);
assertEquals(totalCount,2);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier reader=inputformat.getRecordReader(splits[0],conf,null);
Text rowId=new Text("r1");
AccumuloHiveRow row=new AccumuloHiveRow();
row.add(COLUMN_FAMILY.toString(),NAME.toString(),"brian".getBytes());
row.add(COLUMN_FAMILY.toString(),SID.toString(),parseIntBytes("1"));
row.add(COLUMN_FAMILY.toString(),DEGREES.toString(),parseDoubleBytes("44.5"));
row.add(COLUMN_FAMILY.toString(),MILLIS.toString(),parseLongBytes("555"));
assertTrue(reader.next(rowId,row));
assertEquals(rowId.toString(),row.getRowId());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals("brian".getBytes(),row.getValue(COLUMN_FAMILY,NAME));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,SID));
assertArrayEquals(parseIntBytes("1"),row.getValue(COLUMN_FAMILY,SID));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,DEGREES));
assertArrayEquals(parseDoubleBytes("44.5"),row.getValue(COLUMN_FAMILY,DEGREES));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,MILLIS));
assertArrayEquals(parseLongBytes("555"),row.getValue(COLUMN_FAMILY,MILLIS));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier reader=inputformat.getRecordReader(splits[0],conf,null);
Text rowId=new Text("r1");
AccumuloHiveRow row=new AccumuloHiveRow();
row.setRowId("r1");
assertFalse(reader.next(rowId,row));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier reader=inputformat.getRecordReader(splits[0],conf,null);
Text rowId=new Text("r1");
AccumuloHiveRow row=new AccumuloHiveRow();
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"brian".getBytes());
rowId=new Text("r2");
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"mark".getBytes());
rowId=new Text("r3");
assertTrue(reader.next(rowId,row));
assertEquals(row.getRowId(),rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY,NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY,NAME),"dennis".getBytes());
assertFalse(reader.next(rowId,row));
}
InternalCallVerifier EqualityVerifier settingsOnSplit=((HiveAccumuloSplit)split).getSplit().getIterators();
assertEquals(1,settingsOnSplit.size());
assertEquals(is,settingsOnSplit.get(0));
}
BooleanVerifier EqualityVerifier HybridVerifier kv : scan) {
boolean foundName=false;
boolean foundSid=false;
boolean foundDegrees=false;
boolean foundMillis=false;
SortedMap items=PrimitiveComparisonFilter.decodeRow(kv.getKey(),kv.getValue());
for ( Map.Entry item : items.entrySet()) {
SortedMap nestedItems=PrimitiveComparisonFilter.decodeRow(item.getKey(),item.getValue());
for ( Map.Entry nested : nestedItems.entrySet()) {
if (nested.getKey().getRow().toString().equals("r3")) {
foundDennis=true;
}
if (nested.getKey().getColumnQualifier().equals(NAME)) {
foundName=true;
}
else if (nested.getKey().getColumnQualifier().equals(SID)) {
foundSid=true;
}
else if (nested.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees=true;
}
else if (nested.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis=true;
}
}
}
totalCount++;
assertTrue(foundDegrees & foundMillis & foundName& foundSid);
}
assertTrue(foundDennis);
assertEquals(totalCount,1);
}
EqualityVerifier mappings=new ArrayList();
Set> columns=new HashSet>();
mappings.add(new HiveAccumuloRowIdColumnMapping(AccumuloHiveConstants.ROWID,ColumnEncoding.STRING,"row",TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(new HiveAccumuloColumnMapping("person","name",ColumnEncoding.STRING,"col1",TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(new HiveAccumuloColumnMapping("person","age",ColumnEncoding.STRING,"col2",TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(new HiveAccumuloColumnMapping("person","height",ColumnEncoding.STRING,"col3",TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(new HiveAccumuloColumnMapping("city","name",ColumnEncoding.STRING,"col4",TypeInfoFactory.stringTypeInfo.toString()));
columns.add(new Pair(new Text("person"),new Text("name")));
columns.add(new Pair(new Text("person"),new Text("age")));
columns.add(new Pair(new Text("person"),new Text("height")));
columns.add(new Pair(new Text("city"),new Text("name")));
assertEquals(columns,inputformat.getPairCollection(mappings));
}
Class: org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableOutputFormat APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier recordWriter=outputFormat.getRecordWriter(local,conf,null,null);
List names=Arrays.asList("row","col1","col2");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowID,cf:cq1,cf:cq2");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(names));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
tableProperties.setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE,ColumnEncoding.BINARY.getName());
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,accumuloSerDeParams.getRowIdFactory());
TypeInfo stringTypeInfo=TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(Arrays.asList("row","cq1","cq2"),Arrays.asList(stringTypeInfo,stringTypeInfo,stringTypeInfo),serDeParams.getSeparators(),serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyStruct struct=(LazyStruct)LazyFactory.createLazyObject(structOI);
ByteArrayRef bytes=new ByteArrayRef();
bytes.setData("row value1 value2".getBytes());
struct.init(bytes,0,bytes.getData().length);
Mutation m=serializer.serialize(struct,structOI);
recordWriter.write(new Text(table),m);
recordWriter.close(null);
Iterator> iter=conn.createScanner(table,new Authorizations()).iterator();
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
Entry entry=iter.next();
Key k=entry.getKey();
Value v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq1",k.getColumnQualifier().toString());
Assert.assertEquals("",k.getColumnVisibility().toString());
Assert.assertEquals("value1",new String(v.get()));
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
entry=iter.next();
k=entry.getKey();
v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq2",k.getColumnQualifier().toString());
Assert.assertEquals("",k.getColumnVisibility().toString());
Assert.assertEquals("value2",new String(v.get()));
Assert.assertFalse("Iterator unexpectedly had more data",iter.hasNext());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier > tokens=user1.getTokens();
Assert.assertEquals(1,tokens.size());
Assert.assertEquals(hadoopToken,tokens.iterator().next());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier recordWriter=outputFormat.getRecordWriter(local,conf,null,null);
List names=Arrays.asList("row","col1","col2");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowID,cf:cq1,cf:cq2");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(names));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,accumuloSerDeParams.getRowIdFactory());
TypeInfo stringTypeInfo=TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(Arrays.asList("row","cq1","cq2"),Arrays.asList(stringTypeInfo,stringTypeInfo,stringTypeInfo),serDeParams.getSeparators(),serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyStruct struct=(LazyStruct)LazyFactory.createLazyObject(structOI);
ByteArrayRef bytes=new ByteArrayRef();
bytes.setData("row value1 value2".getBytes());
struct.init(bytes,0,bytes.getData().length);
Mutation m=serializer.serialize(struct,structOI);
recordWriter.write(new Text(table),m);
recordWriter.close(null);
Iterator> iter=conn.createScanner(table,new Authorizations()).iterator();
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
Entry entry=iter.next();
Key k=entry.getKey();
Value v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq1",k.getColumnQualifier().toString());
Assert.assertEquals("",k.getColumnVisibility().toString());
Assert.assertEquals("value1",new String(v.get()));
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
entry=iter.next();
k=entry.getKey();
v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq2",k.getColumnQualifier().toString());
Assert.assertEquals("",k.getColumnVisibility().toString());
Assert.assertEquals("value2",new String(v.get()));
Assert.assertFalse("Iterator unexpectedly had more data",iter.hasNext());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier recordWriter=outputFormat.getRecordWriter(local,conf,null,null);
List names=Arrays.asList("row","col1","col2");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowID,cf:cq1,cf:cq2");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(names));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),new ColumnVisibility("foo"),accumuloSerDeParams.getRowIdFactory());
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(Arrays.asList("row","cq1","cq2"),Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo),serDeParams.getSeparators(),serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyStruct struct=(LazyStruct)LazyFactory.createLazyObject(structOI);
ByteArrayRef bytes=new ByteArrayRef();
bytes.setData("row value1 value2".getBytes());
struct.init(bytes,0,bytes.getData().length);
Mutation m=serializer.serialize(struct,structOI);
recordWriter.write(new Text(table),m);
recordWriter.close(null);
Iterator> iter=conn.createScanner(table,auths).iterator();
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
Entry entry=iter.next();
Key k=entry.getKey();
Value v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq1",k.getColumnQualifier().toString());
Assert.assertEquals("foo",k.getColumnVisibility().toString());
Assert.assertEquals("value1",new String(v.get()));
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
entry=iter.next();
k=entry.getKey();
v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq2",k.getColumnQualifier().toString());
Assert.assertEquals("foo",k.getColumnVisibility().toString());
Assert.assertEquals("value2",new String(v.get()));
Assert.assertFalse("Iterator unexpectedly had more data",iter.hasNext());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier recordWriter=outputFormat.getRecordWriter(local,conf,null,null);
List names=Arrays.asList("row","col1");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowID,cf:*");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(names));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,accumuloSerDeParams.getRowIdFactory());
TypeInfo stringTypeInfo=TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
LazyStringObjectInspector stringOI=(LazyStringObjectInspector)LazyFactory.createLazyObjectInspector(stringTypeInfo,new byte[]{0},0,serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyMapObjectInspector mapOI=LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(stringOI,stringOI,(byte)',',(byte)':',serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(Arrays.asList("row","data"),Arrays.asList(stringOI,mapOI),(byte)' ',serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyStruct struct=(LazyStruct)LazyFactory.createLazyObject(structOI);
ByteArrayRef bytes=new ByteArrayRef();
bytes.setData("row cq1:value1,cq2:value2".getBytes());
struct.init(bytes,0,bytes.getData().length);
Mutation m=serializer.serialize(struct,structOI);
recordWriter.write(new Text(table),m);
recordWriter.close(null);
Iterator> iter=conn.createScanner(table,new Authorizations()).iterator();
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
Entry entry=iter.next();
Key k=entry.getKey();
Value v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq1",k.getColumnQualifier().toString());
Assert.assertEquals(AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,k.getColumnVisibilityParsed());
Assert.assertEquals("value1",new String(v.get()));
Assert.assertTrue("Iterator did not have an element as expected",iter.hasNext());
entry=iter.next();
k=entry.getKey();
v=entry.getValue();
Assert.assertEquals("row",k.getRow().toString());
Assert.assertEquals("cf",k.getColumnFamily().toString());
Assert.assertEquals("cq2",k.getColumnQualifier().toString());
Assert.assertEquals(AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,k.getColumnVisibilityParsed());
Assert.assertEquals("value2",new String(v.get()));
Assert.assertFalse("Iterator unexpectedly had more data",iter.hasNext());
}
Class: org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTypes APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier e : conn.createScanner(tableName,new Authorizations())) {
System.out.println(e);
}
FileInputFormat.addInputPath(conf,new Path("unused"));
InputSplit[] splits=inputformat.getSplits(conf,0);
assertEquals(splits.length,1);
RecordReader reader=inputformat.getRecordReader(splits[0],conf,null);
Text key=reader.createKey();
AccumuloHiveRow value=reader.createValue();
reader.next(key,value);
Assert.assertEquals(13,value.getTuples().size());
Text cfText=new Text(cf), cqHolder=new Text();
cqHolder.set("string");
byte[] valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyStringObjectInspector lazyStringOI=LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false,(byte)0);
LazyString lazyString=(LazyString)LazyFactory.createLazyObject(lazyStringOI);
lazyString.init(byteRef,0,valueBytes.length);
Assert.assertEquals(new Text(stringValue),lazyString.getWritableObject());
cqHolder.set("boolean");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyBooleanObjectInspector lazyBooleanOI=(LazyBooleanObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
LazyBoolean lazyBoolean=(LazyBoolean)LazyFactory.createLazyObject(lazyBooleanOI);
lazyBoolean.init(byteRef,0,valueBytes.length);
Assert.assertEquals(booleanValue,lazyBoolean.getWritableObject().get());
cqHolder.set("tinyint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyByteObjectInspector lazyByteOI=(LazyByteObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
LazyByte lazyByte=(LazyByte)LazyFactory.createLazyObject(lazyByteOI);
lazyByte.init(byteRef,0,valueBytes.length);
Assert.assertEquals(tinyintValue,lazyByte.getWritableObject().get());
cqHolder.set("smallint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyShortObjectInspector lazyShortOI=(LazyShortObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
LazyShort lazyShort=(LazyShort)LazyFactory.createLazyObject(lazyShortOI);
lazyShort.init(byteRef,0,valueBytes.length);
Assert.assertEquals(smallintValue,lazyShort.getWritableObject().get());
cqHolder.set("int");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyIntObjectInspector lazyIntOI=(LazyIntObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
LazyInteger lazyInt=(LazyInteger)LazyFactory.createLazyObject(lazyIntOI);
lazyInt.init(byteRef,0,valueBytes.length);
Assert.assertEquals(intValue,lazyInt.getWritableObject().get());
cqHolder.set("bigint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyLongObjectInspector lazyLongOI=(LazyLongObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
LazyLong lazyLong=(LazyLong)LazyFactory.createLazyObject(lazyLongOI);
lazyLong.init(byteRef,0,valueBytes.length);
Assert.assertEquals(bigintValue,lazyLong.getWritableObject().get());
cqHolder.set("float");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyFloatObjectInspector lazyFloatOI=(LazyFloatObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
LazyFloat lazyFloat=(LazyFloat)LazyFactory.createLazyObject(lazyFloatOI);
lazyFloat.init(byteRef,0,valueBytes.length);
Assert.assertEquals(floatValue,lazyFloat.getWritableObject().get(),0);
cqHolder.set("double");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyDoubleObjectInspector lazyDoubleOI=(LazyDoubleObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
LazyDouble lazyDouble=(LazyDouble)LazyFactory.createLazyObject(lazyDoubleOI);
lazyDouble.init(byteRef,0,valueBytes.length);
Assert.assertEquals(doubleValue,lazyDouble.getWritableObject().get(),0);
cqHolder.set("decimal");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveDecimalObjectInspector lazyDecimalOI=(LazyHiveDecimalObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new DecimalTypeInfo(5,2));
LazyHiveDecimal lazyDecimal=(LazyHiveDecimal)LazyFactory.createLazyObject(lazyDecimalOI);
lazyDecimal.init(byteRef,0,valueBytes.length);
Assert.assertEquals(decimalValue,lazyDecimal.getWritableObject().getHiveDecimal());
cqHolder.set("date");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyDateObjectInspector lazyDateOI=(LazyDateObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME));
LazyDate lazyDate=(LazyDate)LazyFactory.createLazyObject(lazyDateOI);
lazyDate.init(byteRef,0,valueBytes.length);
Assert.assertEquals(dateValue,lazyDate.getWritableObject().get());
cqHolder.set("timestamp");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyTimestampObjectInspector lazyTimestampOI=(LazyTimestampObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME));
LazyTimestamp lazyTimestamp=(LazyTimestamp)LazyFactory.createLazyObject(lazyTimestampOI);
lazyTimestamp.init(byteRef,0,valueBytes.length);
Assert.assertEquals(timestampValue,lazyTimestamp.getWritableObject().getTimestamp());
cqHolder.set("char");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveCharObjectInspector lazyCharOI=(LazyHiveCharObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new CharTypeInfo(4));
LazyHiveChar lazyChar=(LazyHiveChar)LazyFactory.createLazyObject(lazyCharOI);
lazyChar.init(byteRef,0,valueBytes.length);
Assert.assertEquals(charValue,lazyChar.getWritableObject().getHiveChar());
cqHolder.set("varchar");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveVarcharObjectInspector lazyVarcharOI=(LazyHiveVarcharObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new VarcharTypeInfo(7));
LazyHiveVarchar lazyVarchar=(LazyHiveVarchar)LazyFactory.createLazyObject(lazyVarcharOI);
lazyVarchar.init(byteRef,0,valueBytes.length);
Assert.assertEquals(varcharValue.toString(),lazyVarchar.getWritableObject().getHiveVarchar().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier e : conn.createScanner(tableName,new Authorizations())) {
System.out.println(e);
}
FileInputFormat.addInputPath(conf,new Path("unused"));
InputSplit[] splits=inputformat.getSplits(conf,0);
assertEquals(splits.length,1);
RecordReader reader=inputformat.getRecordReader(splits[0],conf,null);
Text key=reader.createKey();
AccumuloHiveRow value=reader.createValue();
reader.next(key,value);
Assert.assertEquals(13,value.getTuples().size());
ByteArrayRef byteRef=new ByteArrayRef();
Text cfText=new Text(cf), cqHolder=new Text();
cqHolder.set("string");
byte[] valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyStringObjectInspector lazyStringOI=LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false,(byte)0);
LazyString lazyString=(LazyString)LazyFactory.createLazyObject(lazyStringOI);
lazyString.init(byteRef,0,valueBytes.length);
Assert.assertEquals(stringValue,lazyString.getWritableObject().toString());
cqHolder.set("boolean");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyBooleanObjectInspector lazyBooleanOI=(LazyBooleanObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
LazyBoolean lazyBoolean=(LazyBoolean)LazyFactory.createLazyPrimitiveBinaryClass(lazyBooleanOI);
lazyBoolean.init(byteRef,0,valueBytes.length);
Assert.assertEquals(booleanValue,lazyBoolean.getWritableObject().get());
cqHolder.set("tinyint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyByteObjectInspector lazyByteOI=(LazyByteObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
LazyByte lazyByte=(LazyByte)LazyFactory.createLazyPrimitiveBinaryClass(lazyByteOI);
lazyByte.init(byteRef,0,valueBytes.length);
Assert.assertEquals(tinyintValue,lazyByte.getWritableObject().get());
cqHolder.set("smallint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyShortObjectInspector lazyShortOI=(LazyShortObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
LazyShort lazyShort=(LazyShort)LazyFactory.createLazyPrimitiveBinaryClass(lazyShortOI);
lazyShort.init(byteRef,0,valueBytes.length);
Assert.assertEquals(smallintValue,lazyShort.getWritableObject().get());
cqHolder.set("int");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyIntObjectInspector lazyIntOI=(LazyIntObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
LazyInteger lazyInt=(LazyInteger)LazyFactory.createLazyPrimitiveBinaryClass(lazyIntOI);
lazyInt.init(byteRef,0,valueBytes.length);
Assert.assertEquals(intValue,lazyInt.getWritableObject().get());
cqHolder.set("bigint");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyLongObjectInspector lazyLongOI=(LazyLongObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
LazyLong lazyLong=(LazyLong)LazyFactory.createLazyPrimitiveBinaryClass(lazyLongOI);
lazyLong.init(byteRef,0,valueBytes.length);
Assert.assertEquals(bigintValue,lazyLong.getWritableObject().get());
cqHolder.set("float");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyFloatObjectInspector lazyFloatOI=(LazyFloatObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
LazyFloat lazyFloat=(LazyFloat)LazyFactory.createLazyPrimitiveBinaryClass(lazyFloatOI);
lazyFloat.init(byteRef,0,valueBytes.length);
Assert.assertEquals(floatValue,lazyFloat.getWritableObject().get(),0);
cqHolder.set("double");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyDoubleObjectInspector lazyDoubleOI=(LazyDoubleObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
LazyDouble lazyDouble=(LazyDouble)LazyFactory.createLazyPrimitiveBinaryClass(lazyDoubleOI);
lazyDouble.init(byteRef,0,valueBytes.length);
Assert.assertEquals(doubleValue,lazyDouble.getWritableObject().get(),0);
cqHolder.set("decimal");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
ByteArrayInputStream bais=new ByteArrayInputStream(valueBytes);
DataInputStream in=new DataInputStream(bais);
decimalWritable.readFields(in);
Assert.assertEquals(decimalValue,decimalWritable.getHiveDecimal());
cqHolder.set("date");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
bais=new ByteArrayInputStream(valueBytes);
in=new DataInputStream(bais);
dateWritable.readFields(in);
Assert.assertEquals(dateValue,dateWritable.get());
cqHolder.set("timestamp");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
bais=new ByteArrayInputStream(valueBytes);
in=new DataInputStream(bais);
timestampWritable.readFields(in);
Assert.assertEquals(timestampValue,timestampWritable.getTimestamp());
cqHolder.set("char");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveCharObjectInspector lazyCharOI=(LazyHiveCharObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new CharTypeInfo(4));
LazyHiveChar lazyChar=(LazyHiveChar)LazyFactory.createLazyObject(lazyCharOI);
lazyChar.init(byteRef,0,valueBytes.length);
Assert.assertEquals(charValue,lazyChar.getWritableObject().getHiveChar());
cqHolder.set("varchar");
valueBytes=value.getValue(cfText,cqHolder);
Assert.assertNotNull(valueBytes);
byteRef.setData(valueBytes);
LazyHiveVarcharObjectInspector lazyVarcharOI=(LazyHiveVarcharObjectInspector)LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new VarcharTypeInfo(7));
LazyHiveVarchar lazyVarchar=(LazyHiveVarchar)LazyFactory.createLazyObject(lazyVarcharOI);
lazyVarchar.init(byteRef,0,valueBytes.length);
Assert.assertEquals(varcharValue.toString(),lazyVarchar.getWritableObject().getHiveVarchar().toString());
}
Class: org.apache.hadoop.hive.accumulo.predicate.TestAccumuloPredicateHandler InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnNames=Arrays.asList("key","column");
List columnTypes=Arrays.asList(TypeInfoFactory.intTypeInfo,TypeInfoFactory.stringTypeInfo);
conf.set(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columnNames));
conf.set(serdeConstants.LIST_COLUMN_TYPES,"int,string");
String columnMappingStr=":rowID#b,cf:f1";
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS,columnMappingStr);
columnMapper=new ColumnMapper(columnMappingStr,ColumnEncoding.STRING.getName(),columnNames,columnTypes);
int intValue=100;
ByteArrayOutputStream baos=new ByteArrayOutputStream();
JavaIntObjectInspector intOI=(JavaIntObjectInspector)PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
LazyUtils.writePrimitive(baos,intValue,intOI);
ExprNodeDesc column=new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo,"key",null,false);
ExprNodeDesc constant=new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo,intValue);
List children=Lists.newArrayList();
children.add(constant);
children.add(column);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPLessThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List ranges=handler.getRanges(conf,columnMapper);
Assert.assertEquals(1,ranges.size());
Assert.assertEquals(new Range(new Text(baos.toByteArray()),false,null,false),ranges.get(0));
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqual(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List sConditions=handler.getSearchConditions(conf);
assertEquals(sConditions.size(),1);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPLessThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(ranges.size(),1);
Range range=ranges.iterator().next();
assertTrue(range.isStartKeyInclusive());
assertFalse(range.isEndKeyInclusive());
assertFalse(range.contains(new Key(new Text("aaa"))));
assertTrue(range.afterEndKey(new Key(new Text("ccccc"))));
assertTrue(range.contains(new Key(new Text("aa"))));
assertTrue(range.afterEndKey(new Key(new Text("aab"))));
assertTrue(range.afterEndKey(new Key(new Text("aaa"))));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqual(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(ranges.size(),1);
Range range=ranges.iterator().next();
assertTrue(range.isStartKeyInclusive());
assertFalse(range.isEndKeyInclusive());
assertTrue(range.contains(new Key(new Text("aaa"))));
assertTrue(range.afterEndKey(new Key(new Text("aab"))));
assertTrue(range.beforeStartKey(new Key(new Text("aa"))));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"bbb");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List ranges=handler.getRanges(conf,columnMapper);
assertEquals(1,ranges.size());
Range range=ranges.get(0);
assertEquals(new Range(new Key("aaa"),true,new Key("bbb"),false),range);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqual(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List sConditions=handler.getSearchConditions(conf);
assertEquals(sConditions.size(),1);
IndexSearchCondition sc=sConditions.get(0);
PushdownTuple tuple=new PushdownTuple(sConditions.get(0),handler.getPrimitiveComparison(sc.getColumnDesc().getTypeString(),sc),handler.getCompareOp(sc.getComparisonOp(),sc));
byte[] expectedVal=new byte[4];
ByteBuffer.wrap(expectedVal).putInt(5);
assertArrayEquals(tuple.getConstVal(),expectedVal);
assertEquals(tuple.getcOpt().getClass(),Equal.class);
assertEquals(tuple.getpCompare().getClass(),IntCompare.class);
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"bbb");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPGreaterThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(0,ranges.size());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnNames=Arrays.asList("key","column");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
conf.set(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columnNames));
conf.set(serdeConstants.LIST_COLUMN_TYPES,"string,string");
String columnMappingStr=":rowID,cf:f1";
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS,columnMappingStr);
columnMapper=new ColumnMapper(columnMappingStr,ColumnEncoding.STRING.getName(),columnNames,columnTypes);
ExprNodeDesc column=new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo,"key",null,false);
ExprNodeDesc constant=new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo,100);
List children=Lists.newArrayList();
children.add(constant);
children.add(column);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPLessThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List ranges=handler.getRanges(conf,columnMapper);
Assert.assertEquals(1,ranges.size());
Assert.assertEquals(new Range(new Text("100"),false,null,false),ranges.get(0));
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(ranges.size(),1);
Range range=ranges.iterator().next();
assertTrue(range.isStartKeyInclusive());
assertFalse(range.isEndKeyInclusive());
assertTrue(range.contains(new Key(new Text("aaa"))));
assertTrue(range.afterEndKey(new Key(new Text("ccccc"))));
assertTrue(range.contains(new Key(new Text("aa"))));
assertTrue(range.afterEndKey(new Key(new Text("aab"))));
assertFalse(range.afterEndKey(new Key(new Text("aaa"))));
}
UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPNotNull(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List sConditions=handler.getSearchConditions(conf);
assertEquals(sConditions.size(),1);
IndexSearchCondition sc=sConditions.get(0);
new PushdownTuple(sc,handler.getPrimitiveComparison(sc.getColumnDesc().getTypeString(),sc),handler.getCompareOp(sc.getComparisonOp(),sc));
fail("Should fail: compare op not registered for index analyzer. Should leave undesirable residual predicate");
}
catch ( RuntimeException e) {
assertTrue(e.getMessage().contains("Unexpected residual predicate: field1 is not null"));
}
catch ( Exception e) {
fail(StringUtils.stringifyException(e));
}
}
BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnNames=Arrays.asList("field1","field2","rid");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.stringTypeInfo);
conf.set(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columnNames));
conf.set(serdeConstants.LIST_COLUMN_TYPES,"string,int,string");
String columnMappingStr="cf:f1,cf:f2,:rowID";
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS,columnMappingStr);
columnMapper=new ColumnMapper(columnMappingStr,ColumnEncoding.STRING.getName(),columnNames,columnTypes);
ExprNodeDesc column=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"field1",null,false);
ExprNodeDesc constant=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"aaa");
List children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo,"field2",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo,5);
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPGreaterThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List iterators=handler.getIterators(conf,columnMapper);
assertEquals(iterators.size(),2);
IteratorSetting is1=iterators.get(0);
IteratorSetting is2=iterators.get(1);
boolean foundQual=false;
boolean foundPCompare=false;
boolean foundCOpt=false;
boolean foundConst=false;
for ( Map.Entry option : is1.getOptions().entrySet()) {
String optKey=option.getKey();
if (optKey.equals(PrimitiveComparisonFilter.COLUMN)) {
foundQual=true;
assertEquals(option.getValue(),"cf:f1");
}
else if (optKey.equals(PrimitiveComparisonFilter.CONST_VAL)) {
foundConst=true;
assertEquals(option.getValue(),new String(Base64.encodeBase64("aaa".getBytes())));
}
else if (optKey.equals(PrimitiveComparisonFilter.COMPARE_OPT_CLASS)) {
foundCOpt=true;
assertEquals(option.getValue(),LessThanOrEqual.class.getName());
}
else if (optKey.equals(PrimitiveComparisonFilter.P_COMPARE_CLASS)) {
foundPCompare=true;
assertEquals(option.getValue(),StringCompare.class.getName());
}
}
assertTrue(foundConst & foundCOpt & foundPCompare& foundQual);
foundQual=false;
foundPCompare=false;
foundCOpt=false;
foundConst=false;
for ( Map.Entry option : is2.getOptions().entrySet()) {
String optKey=option.getKey();
if (optKey.equals(PrimitiveComparisonFilter.COLUMN)) {
foundQual=true;
assertEquals(option.getValue(),"cf:f2");
}
else if (optKey.equals(PrimitiveComparisonFilter.CONST_VAL)) {
foundConst=true;
byte[] intVal=new byte[4];
ByteBuffer.wrap(intVal).putInt(5);
assertEquals(option.getValue(),new String(Base64.encodeBase64(intVal)));
}
else if (optKey.equals(PrimitiveComparisonFilter.COMPARE_OPT_CLASS)) {
foundCOpt=true;
assertEquals(option.getValue(),GreaterThan.class.getName());
}
else if (optKey.equals(PrimitiveComparisonFilter.P_COMPARE_CLASS)) {
foundPCompare=true;
assertEquals(option.getValue(),IntCompare.class.getName());
}
}
assertTrue(foundConst & foundCOpt & foundPCompare& foundQual);
}
UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnNames=Arrays.asList("field1","field2","rid");
List columnTypes=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.stringTypeInfo);
conf.set(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columnNames));
conf.set(serdeConstants.LIST_COLUMN_TYPES,"string,int,string");
String columnMappingStr="cf:f1,cf:f2,:rowID";
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS,columnMappingStr);
columnMapper=new ColumnMapper(columnMappingStr,ColumnEncoding.STRING.getName(),columnNames,columnTypes);
ExprNodeDesc column=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"field1",null,false);
ExprNodeDesc constant=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"aaa");
List children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo,"field2",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo,5);
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPGreaterThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
conf.setBoolean(AccumuloSerDeParameters.ITERATOR_PUSHDOWN_KEY,false);
try {
List iterators=handler.getIterators(conf,columnMapper);
assertEquals(iterators.size(),0);
}
catch ( Exception e) {
fail(StringUtils.stringifyException(e));
}
}
InternalCallVerifier EqualityVerifier NullVerifier ExceptionVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqual(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List sConditions=handler.getSearchConditions(conf);
assertEquals(sConditions.size(),1);
IndexSearchCondition sc=sConditions.get(0);
handler.getPrimitiveComparison(sc.getColumnDesc().getTypeString(),sc);
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPGreaterThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(ranges.size(),1);
Range range=ranges.iterator().next();
assertTrue(range.isStartKeyInclusive());
assertFalse(range.isEndKeyInclusive());
assertFalse(range.contains(new Key(new Text("aaa"))));
assertFalse(range.afterEndKey(new Key(new Text("ccccc"))));
assertTrue(range.contains(new Key(new Text("aab"))));
assertTrue(range.beforeStartKey(new Key(new Text("aa"))));
assertTrue(range.beforeStartKey(new Key(new Text("aaa"))));
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"bbb");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPGreaterThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
try {
List iterators=handler.getIterators(conf,columnMapper);
assertEquals(iterators.size(),0);
}
catch ( SerDeException e) {
StringUtils.stringifyException(e);
}
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeGenericFuncDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
String filterExpr=SerializationUtilities.serializeExpression(node);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
Collection ranges=handler.getRanges(conf,columnMapper);
assertEquals(ranges.size(),1);
Range range=ranges.iterator().next();
assertTrue(range.isStartKeyInclusive());
assertFalse(range.isEndKeyInclusive());
assertTrue(range.contains(new Key(new Text("aaa"))));
assertFalse(range.afterEndKey(new Key(new Text("ccccc"))));
assertTrue(range.contains(new Key(new Text("aab"))));
assertTrue(range.beforeStartKey(new Key(new Text("aa"))));
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"m");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
String filterExpr=SerializationUtilities.serializeExpression(both);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,filterExpr);
List ranges=handler.getRanges(conf,columnMapper);
assertEquals(1,ranges.size());
assertEquals(new Range(new Key("f"),true,new Key("m\0"),false),ranges.get(0));
}
Class: org.apache.hadoop.hive.accumulo.predicate.TestAccumuloRangeGenerator APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"m");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
List expectedRanges=Arrays.asList(new Range(new Key("f"),true,new Key("m\0"),false));
AccumuloRangeGenerator rangeGenerator=new AccumuloRangeGenerator(handler,rowIdMapping,"rid");
Dispatcher disp=new DefaultRuleDispatcher(rangeGenerator,Collections.emptyMap(),null);
GraphWalker ogw=new DefaultGraphWalker(disp);
ArrayList topNodes=new ArrayList();
topNodes.add(both);
HashMap nodeOutput=new HashMap();
try {
ogw.startWalking(topNodes,nodeOutput);
}
catch ( SemanticException ex) {
throw new RuntimeException(ex);
}
Object result=nodeOutput.get(both);
Assert.assertNotNull(result);
Assert.assertTrue("Result from graph walk was not a List",result instanceof List);
@SuppressWarnings("unchecked") List actualRanges=(List)result;
Assert.assertEquals(expectedRanges,actualRanges);
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo,Date.valueOf("2014-07-01"));
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
List expectedRanges=Arrays.asList(new Range(new Key("2014-01-01"),true,new Key("2014-07-01"),false));
AccumuloRangeGenerator rangeGenerator=new AccumuloRangeGenerator(handler,rowIdMapping,"rid");
Dispatcher disp=new DefaultRuleDispatcher(rangeGenerator,Collections.emptyMap(),null);
GraphWalker ogw=new DefaultGraphWalker(disp);
ArrayList topNodes=new ArrayList();
topNodes.add(both);
HashMap nodeOutput=new HashMap();
try {
ogw.startWalking(topNodes,nodeOutput);
}
catch ( SemanticException ex) {
throw new RuntimeException(ex);
}
Object result=nodeOutput.get(both);
Assert.assertNotNull(result);
Assert.assertTrue("Result from graph walk was not a List",result instanceof List);
@SuppressWarnings("unchecked") List actualRanges=(List)result;
Assert.assertEquals(expectedRanges,actualRanges);
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"anythingElse",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"foo");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),bothFilters);
List expectedRanges=Arrays.asList(new Range(new Key("f"),true,null,false));
AccumuloRangeGenerator rangeGenerator=new AccumuloRangeGenerator(handler,rowIdMapping,"rid");
Dispatcher disp=new DefaultRuleDispatcher(rangeGenerator,Collections.emptyMap(),null);
GraphWalker ogw=new DefaultGraphWalker(disp);
ArrayList topNodes=new ArrayList();
topNodes.add(both);
HashMap nodeOutput=new HashMap();
try {
ogw.startWalking(topNodes,nodeOutput);
}
catch ( SemanticException ex) {
throw new RuntimeException(ex);
}
Object result=nodeOutput.get(both);
Assert.assertNotNull(result);
Assert.assertTrue("Result from graph walk was not a List",result instanceof List);
@SuppressWarnings("unchecked") List actualRanges=(List)result;
Assert.assertEquals(expectedRanges,actualRanges);
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"d");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children2);
assertNotNull(node2);
ExprNodeDesc column3=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant3=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"q");
List children3=Lists.newArrayList();
children3.add(column3);
children3.add(constant3);
ExprNodeDesc node3=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children3);
assertNotNull(node3);
List orFilters=Lists.newArrayList();
orFilters.add(node2);
orFilters.add(node3);
ExprNodeGenericFuncDesc orNode=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPOr(),orFilters);
List andFilters=Lists.newArrayList();
andFilters.add(node);
andFilters.add(orNode);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPAnd(),andFilters);
List expectedRanges=Arrays.asList(new Range(new Key("q"),true,null,false));
AccumuloRangeGenerator rangeGenerator=new AccumuloRangeGenerator(handler,rowIdMapping,"rid");
Dispatcher disp=new DefaultRuleDispatcher(rangeGenerator,Collections.emptyMap(),null);
GraphWalker ogw=new DefaultGraphWalker(disp);
ArrayList topNodes=new ArrayList();
topNodes.add(both);
HashMap nodeOutput=new HashMap();
try {
ogw.startWalking(topNodes,nodeOutput);
}
catch ( SemanticException ex) {
throw new RuntimeException(ex);
}
Object result=nodeOutput.get(both);
Assert.assertNotNull(result);
Assert.assertTrue("Result from graph walk was not a List",result instanceof List);
@SuppressWarnings("unchecked") List actualRanges=(List)result;
Assert.assertEquals(expectedRanges,actualRanges);
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier children=Lists.newArrayList();
children.add(column);
children.add(constant);
ExprNodeDesc node=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrGreaterThan(),children);
assertNotNull(node);
ExprNodeDesc column2=new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,"rid",null,false);
ExprNodeDesc constant2=new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo,"m");
List children2=Lists.newArrayList();
children2.add(column2);
children2.add(constant2);
ExprNodeDesc node2=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPEqualOrLessThan(),children2);
assertNotNull(node2);
List bothFilters=Lists.newArrayList();
bothFilters.add(node);
bothFilters.add(node2);
ExprNodeGenericFuncDesc both=new ExprNodeGenericFuncDesc(TypeInfoFactory.stringTypeInfo,new GenericUDFOPOr(),bothFilters);
List expectedRanges=Arrays.asList(new Range());
AccumuloRangeGenerator rangeGenerator=new AccumuloRangeGenerator(handler,rowIdMapping,"rid");
Dispatcher disp=new DefaultRuleDispatcher(rangeGenerator,Collections.emptyMap(),null);
GraphWalker ogw=new DefaultGraphWalker(disp);
ArrayList topNodes=new ArrayList();
topNodes.add(both);
HashMap nodeOutput=new HashMap();
try {
ogw.startWalking(topNodes,nodeOutput);
}
catch ( SemanticException ex) {
throw new RuntimeException(ex);
}
Object result=nodeOutput.get(both);
Assert.assertNotNull(result);
Assert.assertTrue("Result from graph walk was not a List",result instanceof List);
@SuppressWarnings("unchecked") List actualRanges=(List)result;
Assert.assertEquals(expectedRanges,actualRanges);
}
Class: org.apache.hadoop.hive.accumulo.predicate.TestPrimitiveComparisonFilter APIUtilityVerifier IterativeVerifier EqualityVerifier options=new HashMap();
for (int i=0; i < 500; i++) {
String constant=Integer.toString(i);
options.put(PrimitiveComparisonFilter.CONST_VAL,new String(Base64.encodeBase64(constant.getBytes())));
Assert.assertEquals(constant,new String(filter.getConstant(options)));
}
}
IterativeVerifier InternalCallVerifier EqualityVerifier options=new HashMap();
IntWritable writable=new IntWritable();
ByteArrayOutputStream baos=new ByteArrayOutputStream();
DataOutputStream out=new DataOutputStream(baos);
for (int i=0; i < 500; i++) {
writable.set(i);
writable.write(out);
options.put(PrimitiveComparisonFilter.CONST_VAL,new String(Base64.encodeBase64(baos.toByteArray())));
byte[] bytes=filter.getConstant(options);
ByteArrayInputStream bais=new ByteArrayInputStream(bytes);
DataInputStream in=new DataInputStream(bais);
writable.readFields(in);
Assert.assertEquals(i,writable.get());
baos.reset();
}
}
Class: org.apache.hadoop.hive.accumulo.serde.TestAccumuloRowSerializer APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","cq1","cq2","cq3");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.stringTypeInfo);
List typeNames=new ArrayList(types.size());
for ( TypeInfo type : types) {
typeNames.add(type.getTypeName());
}
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,cf:cq1#b,cf:cq2#b,cf:cq3");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(typeNames));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
LazySimpleStructObjectInspector oi=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(columns,types,serDeParams.getSeparators(),serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),new ColumnVisibility(),accumuloSerDeParams.getRowIdFactory());
LazyStruct obj=(LazyStruct)LazyFactory.createLazyObject(oi);
ByteArrayRef byteRef=new ByteArrayRef();
byteRef.setData(new byte[]{'r','o','w','1',' ','1','0',' ','2','0',' ','v','a','l','u','e'});
obj.init(byteRef,0,byteRef.getData().length);
Mutation m=(Mutation)serializer.serialize(obj,oi);
Assert.assertArrayEquals("row1".getBytes(),m.getRow());
List updates=m.getUpdates();
Assert.assertEquals(3,updates.size());
ByteArrayOutputStream baos=new ByteArrayOutputStream();
DataOutputStream out=new DataOutputStream(baos);
ColumnUpdate update=updates.get(0);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq1",new String(update.getColumnQualifier()));
out.writeInt(10);
Assert.assertArrayEquals(baos.toByteArray(),update.getValue());
update=updates.get(1);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq2",new String(update.getColumnQualifier()));
baos.reset();
out.writeInt(20);
Assert.assertArrayEquals(baos.toByteArray(),update.getValue());
update=updates.get(2);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq3",new String(update.getColumnQualifier()));
Assert.assertEquals("value",new String(update.getValue()));
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","cq1","cq2","cq3");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.intTypeInfo,TypeInfoFactory.stringTypeInfo);
List typeNames=new ArrayList(types.size());
for ( TypeInfo type : types) {
typeNames.add(type.getTypeName());
}
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,cf:cq1#b,cf:cq2#b,cf:cq3");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(typeNames));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
LazySimpleStructObjectInspector oi=(LazySimpleStructObjectInspector)LazyFactory.createLazyStructInspector(columns,types,serDeParams.getSeparators(),serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),new ColumnVisibility("foo"),accumuloSerDeParams.getRowIdFactory());
LazyStruct obj=(LazyStruct)LazyFactory.createLazyObject(oi);
ByteArrayRef byteRef=new ByteArrayRef();
byteRef.setData(new byte[]{'r','o','w','1',' ','1','0',' ','2','0',' ','v','a','l','u','e'});
obj.init(byteRef,0,byteRef.getData().length);
Mutation m=(Mutation)serializer.serialize(obj,oi);
Assert.assertArrayEquals("row1".getBytes(),m.getRow());
List updates=m.getUpdates();
Assert.assertEquals(3,updates.size());
ByteArrayOutputStream baos=new ByteArrayOutputStream();
DataOutputStream out=new DataOutputStream(baos);
ColumnUpdate update=updates.get(0);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq1",new String(update.getColumnQualifier()));
Assert.assertEquals("foo",new String(update.getColumnVisibility()));
out.writeInt(10);
Assert.assertArrayEquals(baos.toByteArray(),update.getValue());
update=updates.get(1);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq2",new String(update.getColumnQualifier()));
Assert.assertEquals("foo",new String(update.getColumnVisibility()));
baos.reset();
out.writeInt(20);
Assert.assertArrayEquals(baos.toByteArray(),update.getValue());
update=updates.get(2);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq3",new String(update.getColumnQualifier()));
Assert.assertEquals("foo",new String(update.getColumnVisibility()));
Assert.assertEquals("value",new String(update.getValue()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","col");
List types=Arrays.asList(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo));
List typeNames=new ArrayList(types.size());
for ( TypeInfo type : types) {
typeNames.add(type.getTypeName());
}
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,cf:*");
tableProperties.setProperty(serdeConstants.FIELD_DELIM," ");
tableProperties.setProperty(serdeConstants.COLLECTION_DELIM,",");
tableProperties.setProperty(serdeConstants.MAPKEY_DELIM,":");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(typeNames));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
TypeInfo stringTypeInfo=TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
LazyStringObjectInspector stringOI=(LazyStringObjectInspector)LazyFactory.createLazyObjectInspector(stringTypeInfo,new byte[]{0},0,serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyMapObjectInspector mapOI=LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(stringOI,stringOI,(byte)',',(byte)':',serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columns,Arrays.asList(stringOI,mapOI),(byte)' ',serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),new ColumnVisibility(),accumuloSerDeParams.getRowIdFactory());
LazyStruct obj=(LazyStruct)LazyFactory.createLazyObject(structOI);
ByteArrayRef byteRef=new ByteArrayRef();
byteRef.setData("row1 cq1:10,cq2:20,cq3:value".getBytes());
obj.init(byteRef,0,byteRef.getData().length);
Mutation m=(Mutation)serializer.serialize(obj,structOI);
Assert.assertArrayEquals("row1".getBytes(),m.getRow());
List updates=m.getUpdates();
Assert.assertEquals(3,updates.size());
ColumnUpdate update=updates.get(0);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq1",new String(update.getColumnQualifier()));
Assert.assertEquals("10",new String(update.getValue()));
update=updates.get(1);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq2",new String(update.getColumnQualifier()));
Assert.assertEquals("20",new String(update.getValue()));
update=updates.get(2);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq3",new String(update.getColumnQualifier()));
Assert.assertEquals("value",new String(update.getValue()));
}
Class: org.apache.hadoop.hive.accumulo.serde.TestAccumuloSerDe IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ,string");
properties.setProperty(DelimitedAccumuloRowIdFactory.ACCUMULO_COMPOSITE_DELIMITER,"_");
properties.setProperty(AccumuloSerDeParameters.COMPOSITE_ROWID_FACTORY,DelimitedAccumuloRowIdFactory.class.getName());
serde.initialize(conf,properties);
AccumuloHiveRow row=new AccumuloHiveRow();
row.setRowId("p1_p2_p3");
row.add("cf","f1","v1".getBytes());
Object obj=serde.deserialize(row);
assertTrue(obj instanceof LazyAccumuloRow);
LazyAccumuloRow lazyRow=(LazyAccumuloRow)obj;
Object field0=lazyRow.getField(0);
assertNotNull(field0);
assertTrue(field0 instanceof LazyStruct);
LazyStruct struct=(LazyStruct)field0;
List fields=struct.getFieldsAsList();
assertEquals(3,fields.size());
for (int i=0; i < fields.size(); i++) {
assertEquals(LazyString.class,fields.get(i).getClass());
assertEquals("p" + (i + 1),fields.get(i).toString());
}
Object field1=lazyRow.getField(1);
assertNotNull(field1);
assertTrue("Expected instance of LazyString but was " + field1.getClass(),field1 instanceof LazyString);
assertEquals(field1.toString(),"v1");
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier columns=Arrays.asList("row","col");
List structColNames=Arrays.asList("map1","map2");
TypeInfo mapTypeInfo=TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo,TypeInfoFactory.stringTypeInfo);
List types=Arrays.asList(TypeInfoFactory.getStructTypeInfo(structColNames,Arrays.asList(mapTypeInfo,mapTypeInfo)),TypeInfoFactory.stringTypeInfo);
Properties tableProperties=new Properties();
tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS,":rowid,cf:cq");
tableProperties.setProperty(serdeConstants.LIST_COLUMNS,Joiner.on(',').join(columns));
tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES,Joiner.on(',').join(types));
AccumuloSerDeParameters accumuloSerDeParams=new AccumuloSerDeParameters(new Configuration(),tableProperties,AccumuloSerDe.class.getSimpleName());
LazySerDeParameters serDeParams=accumuloSerDeParams.getSerDeParameters();
byte[] seps=serDeParams.getSeparators();
TypeInfo stringTypeInfo=TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
LazyStringObjectInspector stringOI=(LazyStringObjectInspector)LazyFactory.createLazyObjectInspector(stringTypeInfo,new byte[]{0},0,serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazyMapObjectInspector mapOI=LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(stringOI,stringOI,seps[3],seps[4],serDeParams.getNullSequence(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazySimpleStructObjectInspector rowStructOI=(LazySimpleStructObjectInspector)LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(structColNames,Arrays.asList(mapOI,mapOI),(byte)seps[2],serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columns,Arrays.asList(rowStructOI,stringOI),seps[1],serDeParams.getNullSequence(),serDeParams.isLastColumnTakesRest(),serDeParams.isEscaped(),serDeParams.getEscapeChar());
AccumuloRowSerializer serializer=new AccumuloRowSerializer(0,serDeParams,accumuloSerDeParams.getColumnMappings(),new ColumnVisibility(),accumuloSerDeParams.getRowIdFactory());
Map map1=new HashMap(), map2=new HashMap();
map1.put("key10","value10");
map1.put("key11","value11");
map2.put("key20","value20");
map2.put("key21","value21");
ByteArrayRef byteRef=new ByteArrayRef();
String accumuloRow="key10\5value10\4key11\5value11\3key20\5value20\4key21\5value21";
LazyStruct entireStruct=(LazyStruct)LazyFactory.createLazyObject(structOI);
byteRef.setData((accumuloRow + "\2foo").getBytes());
entireStruct.init(byteRef,0,byteRef.getData().length);
Mutation m=serializer.serialize(entireStruct,structOI);
Assert.assertArrayEquals(accumuloRow.getBytes(),m.getRow());
Assert.assertEquals(1,m.getUpdates().size());
ColumnUpdate update=m.getUpdates().get(0);
Assert.assertEquals("cf",new String(update.getColumnFamily()));
Assert.assertEquals("cq",new String(update.getColumnQualifier()));
Assert.assertEquals("foo",new String(update.getValue()));
AccumuloHiveRow haRow=new AccumuloHiveRow(new String(m.getRow()));
haRow.add("cf","cq","foo".getBytes());
LazyAccumuloRow lazyAccumuloRow=new LazyAccumuloRow(structOI);
lazyAccumuloRow.init(haRow,accumuloSerDeParams.getColumnMappings(),accumuloSerDeParams.getRowIdFactory());
List objects=lazyAccumuloRow.getFieldsAsList();
Assert.assertEquals(2,objects.size());
Assert.assertEquals("foo",objects.get(1).toString());
LazyStruct rowStruct=(LazyStruct)objects.get(0);
List rowObjects=rowStruct.getFieldsAsList();
Assert.assertEquals(2,rowObjects.size());
LazyMap rowMap=(LazyMap)rowObjects.get(0);
Map actualMap=rowMap.getMap();
System.out.println("Actual map 1: " + actualMap);
Map actualStringMap=new HashMap();
for ( Entry entry : actualMap.entrySet()) {
actualStringMap.put(entry.getKey().toString(),entry.getValue().toString());
}
Assert.assertEquals(map1,actualStringMap);
rowMap=(LazyMap)rowObjects.get(1);
actualMap=rowMap.getMap();
System.out.println("Actual map 2: " + actualMap);
actualStringMap=new HashMap();
for ( Entry entry : actualMap.entrySet()) {
actualStringMap.put(entry.getKey().toString(),entry.getValue().toString());
}
Assert.assertEquals(map2,actualStringMap);
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ");
properties.setProperty(serdeConstants.COLLECTION_DELIM,":");
properties.setProperty(serdeConstants.MAPKEY_DELIM,"=");
char collectionSeparator=':', kvSeparator='=';
serde.initialize(conf,properties);
AccumuloHiveRow row=new AccumuloHiveRow();
row.setRowId("r1");
row.add("cf","vals",("k1" + kvSeparator + "v1"+ collectionSeparator+ "k2"+ kvSeparator+ "v2"+ collectionSeparator+ "k3"+ kvSeparator+ "v3").getBytes());
Object obj=serde.deserialize(row);
assertNotNull(obj);
assertTrue(obj instanceof LazyAccumuloRow);
LazyAccumuloRow lazyRow=(LazyAccumuloRow)obj;
Object field0=lazyRow.getField(0);
assertNotNull(field0);
assertTrue(field0 instanceof LazyString);
assertEquals(row.getRowId(),((LazyString)field0).getWritableObject().toString());
Object field1=lazyRow.getField(1);
assertNotNull(field1);
assertTrue(field1 instanceof LazyMap);
LazyMap map=(LazyMap)field1;
Map untypedMap=map.getMap();
assertEquals(3,map.getMapSize());
Set expectedKeys=new HashSet();
expectedKeys.add("k1");
expectedKeys.add("k2");
expectedKeys.add("k3");
for ( Entry entry : untypedMap.entrySet()) {
assertNotNull(entry.getKey());
assertTrue(entry.getKey() instanceof LazyString);
LazyString key=(LazyString)entry.getKey();
assertNotNull(entry.getValue());
assertTrue(entry.getValue() instanceof LazyString);
LazyString value=(LazyString)entry.getValue();
String strKey=key.getWritableObject().toString(), strValue=value.getWritableObject().toString();
assertTrue(expectedKeys.remove(strKey));
assertEquals(2,strValue.length());
assertTrue(strValue.startsWith("v"));
assertTrue(strValue.endsWith(Character.toString(strKey.charAt(1))));
}
assertTrue("Did not find expected keys: " + expectedKeys,expectedKeys.isEmpty());
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ");
properties.setProperty(serdeConstants.COLLECTION_DELIM,":");
char separator=':';
serde.initialize(conf,properties);
AccumuloHiveRow row=new AccumuloHiveRow();
row.setRowId("r1");
row.add("cf","vals",("value1" + separator + "value2"+ separator+ "value3").getBytes());
Object obj=serde.deserialize(row);
assertNotNull(obj);
assertTrue(obj instanceof LazyAccumuloRow);
LazyAccumuloRow lazyRow=(LazyAccumuloRow)obj;
Object field0=lazyRow.getField(0);
assertNotNull(field0);
assertTrue(field0 instanceof LazyString);
assertEquals(row.getRowId(),((LazyString)field0).getWritableObject().toString());
Object field1=lazyRow.getField(1);
assertNotNull(field1);
assertTrue(field1 instanceof LazyArray);
LazyArray array=(LazyArray)field1;
List values=array.getList();
assertEquals(3,values.size());
for (int i=0; i < 3; i++) {
Object o=values.get(i);
assertNotNull(o);
assertTrue(o instanceof LazyString);
assertEquals("value" + (i + 1),((LazyString)o).getWritableObject().toString());
}
}
Class: org.apache.hadoop.hive.accumulo.serde.TestAccumuloSerDeParameters APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.accumulo.serde.TestDefaultAccumuloRowIdFactory APIUtilityVerifier InternalCallVerifier EqualityVerifier ,map");
accumuloSerDe.initialize(conf,properties);
AccumuloRowIdFactory factory=accumuloSerDe.getParams().getRowIdFactory();
List columnTypes=accumuloSerDe.getParams().getHiveColumnTypes();
ColumnMapper mapper=accumuloSerDe.getParams().getColumnMapper();
LazySerDeParameters serDeParams=accumuloSerDe.getParams().getSerDeParameters();
List OIs=accumuloSerDe.getColumnObjectInspectors(columnTypes,serDeParams,mapper.getColumnMappings(),factory);
Assert.assertEquals(2,OIs.size());
Assert.assertEquals(LazySimpleStructObjectInspector.class,OIs.get(0).getClass());
Assert.assertEquals(LazyMapObjectInspector.class,OIs.get(1).getClass());
LazySimpleStructObjectInspector structOI=(LazySimpleStructObjectInspector)OIs.get(0);
Assert.assertEquals(2,(int)structOI.getSeparator());
LazyMapObjectInspector mapOI=(LazyMapObjectInspector)OIs.get(1);
Assert.assertEquals(2,(int)mapOI.getItemSeparator());
Assert.assertEquals(3,(int)mapOI.getKeyValueSeparator());
}
InternalCallVerifier EqualityVerifier columnTypes=accumuloSerDe.getParams().getHiveColumnTypes();
ColumnMapper mapper=accumuloSerDe.getParams().getColumnMapper();
LazySerDeParameters serDeParams=accumuloSerDe.getParams().getSerDeParameters();
List OIs=accumuloSerDe.getColumnObjectInspectors(columnTypes,serDeParams,mapper.getColumnMappings(),factory);
Assert.assertEquals(2,OIs.size());
Assert.assertEquals(LazyStringObjectInspector.class,OIs.get(0).getClass());
Assert.assertEquals(LazyIntObjectInspector.class,OIs.get(1).getClass());
}
Class: org.apache.hadoop.hive.cli.TestCliSessionState EqualityVerifier Class: org.apache.hadoop.hive.cli.TestOptionsProcessor BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.cli.TestRCFileCat APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.common.TestValidReadTxnList InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.common.metrics.TestLegacyMetrics APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier (){
@Override public Void call() throws Exception {
testScopeImpl(n);
return null;
}
}
);
}
executorService.shutdown();
assertTrue(executorService.awaitTermination(periodMs * 3 * threads,TimeUnit.MILLISECONDS));
fooScope=(LegacyMetrics.LegacyMetricsScope)metrics.getStoredScope(scopeName);
assertEquals(Long.valueOf(3 * threads),fooScope.getNumCounter());
assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs * threads);
Double avgT=(Double)metrics.get("foo.avg_t");
assertTrue(avgT.doubleValue() > periodMs);
metrics.endStoredScope(scopeName);
}
APIUtilityVerifier BranchVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier (){
@Override public Long call() throws Exception {
Long num=fooScope.getNumCounter();
return num;
}
}
);
expectIOE(new Callable(){
@Override public Long call() throws Exception {
Long time=fooScope.getTimeCounter();
return time;
}
}
);
expectIOE(new Callable(){
@Override public Void call() throws Exception {
fooScope.open();
return null;
}
}
);
assertSame(fooScope,metrics.getStoredScope(scopeName));
Thread.sleep(periodMs + 1);
metrics.endStoredScope(scopeName);
expectIOE(new Callable(){
@Override public Void call() throws Exception {
metrics.endStoredScope(scopeName);
return null;
}
}
);
assertEquals(Long.valueOf(1),fooScope.getNumCounter());
final long t1=fooScope.getTimeCounter().longValue();
assertTrue(t1 > periodMs);
assertSame(fooScope,metrics.getStoredScope(scopeName));
metrics.startStoredScope(scopeName);
expectIOE(new Callable(){
@Override public Void call() throws Exception {
metrics.startStoredScope(scopeName);
return null;
}
}
);
assertEquals(Long.valueOf(1),fooScope.getNumCounter());
assertEquals(t1,fooScope.getTimeCounter().longValue());
assertSame(fooScope,metrics.getStoredScope(scopeName));
Thread.sleep(periodMs + 1);
fooScope.reopen();
assertEquals(Long.valueOf(2),fooScope.getNumCounter());
assertTrue(fooScope.getTimeCounter().longValue() > 2 * periodMs);
Thread.sleep(periodMs + 1);
fooScope.close();
assertEquals(Long.valueOf(3),fooScope.getNumCounter());
assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs);
Double avgT=(Double)metrics.get("foo.avg_t");
assertTrue(avgT.doubleValue() > periodMs);
}
Class: org.apache.hadoop.hive.common.metrics.metrics2.TestCodahaleMetrics APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier (){
@Override public Void call() throws Exception {
MetricsFactory.getInstance().startStoredScope("method2");
MetricsFactory.getInstance().endStoredScope("method2");
return null;
}
}
);
}
executorService.shutdown();
assertTrue(executorService.awaitTermination(10000,TimeUnit.MILLISECONDS));
Timer timer=metricRegistry.getTimers().get("api_method2");
Assert.assertEquals(4,timer.getCount());
Assert.assertTrue(timer.getMeanRate() > 0);
}
Class: org.apache.hadoop.hive.common.type.TestDecimal128 InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.common.type.TestHiveChar InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.common.type.TestHiveDecimal EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier EqualityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier EqualityVerifier EqualityVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.common.type.TestHiveIntervalDayTime UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.common.type.TestHiveIntervalYearMonth InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.common.type.TestHiveVarchar BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.common.type.TestSignedInt128 EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.common.type.TestSqlMathUtil APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.common.type.TestUnsignedInt128 InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.conf.TestHiveConf UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.conf.TestVariableSubstitution InternalCallVerifier EqualityVerifier getHiveVariable(){
return TestVariableSubstitution.getMySource().map;
}
}
);
String v=variableSubstitution.substitute(new HiveConf(),"${a}");
Assert.assertEquals("${a}",v);
TestVariableSubstitution.getMySource().put("a","b");
v=variableSubstitution.substitute(new HiveConf(),"${a}");
Assert.assertEquals("b",v);
}
Class: org.apache.hadoop.hive.hbase.TestPutResultWritable InternalCallVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.hooks.TestHs2Hooks BooleanVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.llap.cache.TestIncrementalObjectSizeEstimator APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.llap.cache.TestLowLevelCacheImpl APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier rdmCall=new Callable(){
public Long call(){
int gets=0, puts=0;
try {
Random rdm=new Random(1234 + Thread.currentThread().getId());
syncThreadStart(cdlIn,cdlOut);
for (int i=0; i < 20000; ++i) {
boolean isGet=rdm.nextBoolean(), isFn1=rdm.nextBoolean();
long fileName=isFn1 ? fn1 : fn2;
int fileIndex=isFn1 ? 1 : 2;
int count=rdm.nextInt(offsetsToUse);
if (isGet) {
int[] offsets=new int[count];
count=generateOffsets(offsetsToUse,rdm,offsets);
CreateHelper list=new CreateHelper();
for (int j=0; i < count; ++i) {
list.addOrMerge(offsets[j],offsets[j] + 1,true,false);
}
DiskRangeList iter=cache.getFileData(fileName,list.get(),0,testFactory,null,null);
int j=-1;
while (iter != null) {
++j;
if (!(iter instanceof CacheChunk)) {
iter=iter.next;
continue;
}
++gets;
LlapDataBuffer result=(LlapDataBuffer)((CacheChunk)iter).getBuffer();
assertEquals(makeFakeArenaIndex(fileIndex,offsets[j]),result.arenaIndex);
cache.decRefBuffer(result);
iter=iter.next;
}
}
else {
DiskRange[] ranges=new DiskRange[count];
int[] offsets=new int[count];
for (int j=0; j < count; ++j) {
int next=rdm.nextInt(offsetsToUse);
ranges[j]=dr(next,next + 1);
offsets[j]=next;
}
MemoryBuffer[] buffers=new MemoryBuffer[count];
for (int j=0; j < offsets.length; ++j) {
LlapDataBuffer buf=LowLevelCacheImpl.allocateFake();
buf.arenaIndex=makeFakeArenaIndex(fileIndex,offsets[j]);
buffers[j]=buf;
}
long[] mask=cache.putFileData(fileName,ranges,buffers,0,Priority.NORMAL,null);
puts+=buffers.length;
long maskVal=0;
if (mask != null) {
assertEquals(1,mask.length);
maskVal=mask[0];
}
for (int j=0; j < offsets.length; ++j) {
LlapDataBuffer buf=(LlapDataBuffer)(buffers[j]);
if ((maskVal & 1) == 1) {
assertEquals(makeFakeArenaIndex(fileIndex,offsets[j]),buf.arenaIndex);
}
maskVal>>=1;
cache.decRefBuffer(buf);
}
}
}
}
finally {
rdmsDone.incrementAndGet();
}
return (((long)gets) << 32) | puts;
}
private int makeFakeArenaIndex( int fileIndex, long offset){
return (int)((fileIndex << 16) + offset);
}
}
;
FutureTask evictionTask=new FutureTask(new Callable(){
public Integer call(){
boolean isFirstFile=false;
Random rdm=new Random(1234 + Thread.currentThread().getId());
int evictions=0;
syncThreadStart(cdlIn,cdlOut);
while (rdmsDone.get() < 3) {
DiskRangeList head=new DiskRangeList(0,offsetsToUse + 1);
isFirstFile=!isFirstFile;
long fileId=isFirstFile ? fn1 : fn2;
head=cache.getFileData(fileId,head,0,testFactory,null,null);
DiskRange[] results=head.listToArray();
int startIndex=rdm.nextInt(results.length), index=startIndex;
LlapDataBuffer victim=null;
do {
DiskRange r=results[index];
if (r instanceof CacheChunk) {
LlapDataBuffer result=(LlapDataBuffer)((CacheChunk)r).getBuffer();
cache.decRefBuffer(result);
if (victim == null && result.invalidate()) {
++evictions;
victim=result;
}
}
++index;
if (index == results.length) index=0;
}
while (index != startIndex);
if (victim == null) continue;
cache.notifyEvicted(victim);
}
return evictions;
}
}
);
FutureTask rdmTask1=new FutureTask(rdmCall), rdmTask2=new FutureTask(rdmCall), rdmTask3=new FutureTask(rdmCall);
Executor threadPool=Executors.newFixedThreadPool(4);
threadPool.execute(rdmTask1);
threadPool.execute(rdmTask2);
threadPool.execute(rdmTask3);
threadPool.execute(evictionTask);
try {
cdlIn.await();
cdlOut.countDown();
long result1=rdmTask1.get(), result2=rdmTask2.get(), result3=rdmTask3.get();
int evictions=evictionTask.get();
LOG.info("MTT test: task 1: " + descRdmTask(result1) + ", task 2: "+ descRdmTask(result2)+ ", task 3: "+ descRdmTask(result3)+ "; "+ evictions+ " evictions");
}
catch ( Throwable t) {
throw new RuntimeException(t);
}
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.llap.cache.TestLowLevelLrfuCachePolicy BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.llap.cache.TestOrcMetadataCache InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.llap.daemon.impl.TestLlapDaemonProtocolServerImpl APIUtilityVerifier InternalCallVerifier EqualityVerifier (),new AtomicReference(),rpcPort,rpcPort + 1);
when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn(SubmitWorkResponseProto.newBuilder().setSubmissionState(SubmissionStateProto.ACCEPTED).build());
try {
server.init(new Configuration());
server.start();
InetSocketAddress serverAddr=server.getBindAddress();
LlapProtocolBlockingPB client=new LlapProtocolClientImpl(new Configuration(),serverAddr.getHostName(),serverAddr.getPort(),null,null);
SubmitWorkResponseProto responseProto=client.submitWork(null,SubmitWorkRequestProto.newBuilder().setAmHost("amhost").setAmPort(2000).build());
assertEquals(responseProto.getSubmissionState().name(),SubmissionStateProto.ACCEPTED.name());
}
finally {
server.stop();
}
}
Class: org.apache.hadoop.hive.llap.daemon.impl.TestQueryIdentifier IterativeVerifier BranchVerifier EqualityVerifier Class: org.apache.hadoop.hive.llap.daemon.impl.TestTaskExecutorService BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier queue=new PriorityBlockingQueue<>(4,new TaskExecutorService.PreemptionQueueComparator());
queue.offer(r1);
assertEquals(r1,queue.peek());
queue.offer(r2);
assertEquals(r1,queue.peek());
queue.offer(r3);
assertEquals(r1,queue.peek());
queue.offer(r4);
assertEquals(r1,queue.take());
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r4,queue.take());
}
Class: org.apache.hadoop.hive.llap.daemon.impl.comparator.TestFirstInFirstOutComparator InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r4,queue.peek());
assertEquals(r1,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r4,queue.take());
assertEquals(r3,queue.take());
assertEquals(r2,queue.take());
r1=createTaskWrapper(createRequest(1,2,5,100),true,100000);
r2=createTaskWrapper(createRequest(2,4,4,200),true,100000);
r3=createTaskWrapper(createRequest(3,6,3,300),true,1000000);
r4=createTaskWrapper(createRequest(4,8,2,400),true,1000000);
r5=createTaskWrapper(createRequest(5,10,1,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r4,queue.peek());
assertEquals(r1,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r4,queue.take());
assertEquals(r3,queue.take());
assertEquals(r2,queue.take());
r1=createTaskWrapper(createRequest(1,1,5,100),true,100000);
r2=createTaskWrapper(createRequest(2,1,4,200),false,100000);
r3=createTaskWrapper(createRequest(3,1,3,300),true,1000000);
r4=createTaskWrapper(createRequest(4,1,2,400),false,1000000);
r5=createTaskWrapper(createRequest(5,10,1,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r3,queue.peek());
assertEquals(r2,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
assertEquals(r4,queue.take());
r1=createTaskWrapper(createRequest(1,2,5,100),true,100000);
r2=createTaskWrapper(createRequest(2,4,4,200),false,100000);
r3=createTaskWrapper(createRequest(3,6,3,300),true,1000000);
r4=createTaskWrapper(createRequest(4,8,2,400),false,1000000);
r5=createTaskWrapper(createRequest(5,10,1,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r3,queue.peek());
assertEquals(r2,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
assertEquals(r4,queue.take());
r1=createTaskWrapper(createRequest(1,2,5,100),true,100000);
r2=createTaskWrapper(createRequest(2,4,4,200),false,100000);
r3=createTaskWrapper(createRequest(3,6,3,300),false,1000000);
r4=createTaskWrapper(createRequest(4,8,2,400),false,1000000);
r5=createTaskWrapper(createRequest(5,10,1,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r2,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r1,queue.take());
assertEquals(r4,queue.take());
assertEquals(r3,queue.take());
r1=createTaskWrapper(createRequest(1,2,5,100),false,100000);
r2=createTaskWrapper(createRequest(2,4,4,200),true,100000);
r3=createTaskWrapper(createRequest(3,6,3,300),true,1000000);
r4=createTaskWrapper(createRequest(4,8,2,400),true,1000000);
r5=createTaskWrapper(createRequest(5,10,1,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r4,queue.peek());
assertEquals(r1,queue.offer(r5));
assertEquals(r5,queue.take());
assertEquals(r4,queue.take());
assertEquals(r3,queue.take());
assertEquals(r2,queue.take());
r1=createTaskWrapper(createRequest(1,2,5,100),false,100000);
r2=createTaskWrapper(createRequest(2,4,4,200),true,100000);
r3=createTaskWrapper(createRequest(3,6,3,300),true,1000000);
r4=createTaskWrapper(createRequest(4,8,2,400),true,1000000);
r5=createTaskWrapper(createRequest(5,10,2,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r3,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r4,queue.peek());
assertEquals(r1,queue.offer(r5));
assertEquals(r4,queue.take());
assertEquals(r5,queue.take());
assertEquals(r3,queue.take());
assertEquals(r2,queue.take());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertNull(queue.offer(r2));
assertNull(queue.offer(r3));
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new FirstInFirstOutComparator(),4);
assertNull(queue.offer(r1));
assertNull(queue.offer(r2));
assertNull(queue.offer(r3));
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
}
Class: org.apache.hadoop.hive.llap.daemon.impl.comparator.TestShortestJobFirstComparator InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r5,queue.offer(r5));
assertEquals(r1,queue.take());
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r4,queue.take());
r1=createTaskWrapper(createSubmitWorkRequestProto(1,2,100),true,100000);
r2=createTaskWrapper(createSubmitWorkRequestProto(2,4,200),true,100000);
r3=createTaskWrapper(createSubmitWorkRequestProto(3,6,300),true,1000000);
r4=createTaskWrapper(createSubmitWorkRequestProto(4,8,400),true,1000000);
r5=createTaskWrapper(createSubmitWorkRequestProto(5,10,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r5,queue.offer(r5));
assertEquals(r1,queue.take());
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r4,queue.take());
r1=createTaskWrapper(createSubmitWorkRequestProto(1,1,100),true,100000);
r2=createTaskWrapper(createSubmitWorkRequestProto(2,1,200),false,100000);
r3=createTaskWrapper(createSubmitWorkRequestProto(3,1,300),true,1000000);
r4=createTaskWrapper(createSubmitWorkRequestProto(4,1,400),false,1000000);
r5=createTaskWrapper(createSubmitWorkRequestProto(5,10,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r4,queue.offer(r5));
assertEquals(r1,queue.take());
assertEquals(r3,queue.take());
assertEquals(r5,queue.take());
assertEquals(r2,queue.take());
r1=createTaskWrapper(createSubmitWorkRequestProto(1,2,100),true,100000);
r2=createTaskWrapper(createSubmitWorkRequestProto(2,4,200),false,100000);
r3=createTaskWrapper(createSubmitWorkRequestProto(3,6,300),true,1000000);
r4=createTaskWrapper(createSubmitWorkRequestProto(4,8,400),false,1000000);
r5=createTaskWrapper(createSubmitWorkRequestProto(5,10,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r4,queue.offer(r5));
assertEquals(r1,queue.take());
assertEquals(r3,queue.take());
assertEquals(r5,queue.take());
assertEquals(r2,queue.take());
r1=createTaskWrapper(createSubmitWorkRequestProto(1,2,100),true,100000);
r2=createTaskWrapper(createSubmitWorkRequestProto(2,4,200),false,100000);
r3=createTaskWrapper(createSubmitWorkRequestProto(3,6,300),false,1000000);
r4=createTaskWrapper(createSubmitWorkRequestProto(4,8,400),false,1000000);
r5=createTaskWrapper(createSubmitWorkRequestProto(5,10,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r1,queue.peek());
assertEquals(r4,queue.offer(r5));
assertEquals(r1,queue.take());
assertEquals(r5,queue.take());
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
r1=createTaskWrapper(createSubmitWorkRequestProto(1,2,100),false,100000);
r2=createTaskWrapper(createSubmitWorkRequestProto(2,4,200),true,100000);
r3=createTaskWrapper(createSubmitWorkRequestProto(3,6,300),true,1000000);
r4=createTaskWrapper(createSubmitWorkRequestProto(4,8,400),true,1000000);
r5=createTaskWrapper(createSubmitWorkRequestProto(5,10,500),true,1000000);
queue=new EvictingPriorityBlockingQueue(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertEquals(r1,queue.peek());
assertNull(queue.offer(r2));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r3));
assertEquals(r2,queue.peek());
assertNull(queue.offer(r4));
assertEquals(r2,queue.peek());
assertEquals(r1,queue.offer(r5));
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r4,queue.take());
assertEquals(r5,queue.take());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertNull(queue.offer(r2));
assertNull(queue.offer(r3));
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier queue=new EvictingPriorityBlockingQueue<>(new ShortestJobFirstComparator(),4);
assertNull(queue.offer(r1));
assertNull(queue.offer(r2));
assertNull(queue.offer(r3));
assertEquals(r2,queue.take());
assertEquals(r3,queue.take());
assertEquals(r1,queue.take());
}
Class: org.apache.hadoop.hive.llap.tez.TestLlapDaemonProtocolClientProxy InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.llap.tezplugins.TestLlapTaskCommunicator InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.llap.tezplugins.TestLlapTaskSchedulerService InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier argumentCaptor=ArgumentCaptor.forClass(Container.class);
verify(tsWrapper.mockAppCallback,times(3)).taskAllocated(any(Object.class),any(Object.class),argumentCaptor.capture());
assertEquals(0,tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
assertEquals(6,tsWrapper.ts.dagStats.numTotalAllocations);
}
finally {
tsWrapper.shutdown();
}
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.metastore.TestAggregateStatsCache APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier partNames=preparePartNames(tables.get(0),1,9);
BloomFilter bloomFilter=prepareBloomFilter(partNames);
String tblName=tables.get(0);
String colName=tabCols.get(0);
int highVal=100, lowVal=10, numDVs=50, numNulls=5;
ColumnStatisticsObj aggrColStats=getDummyLongColStat(colName,highVal,lowVal,numDVs,numNulls);
cache.add(DB_NAME,tblName,colName,10,aggrColStats,bloomFilter);
AggrColStats aggrStatsCached=cache.get(DB_NAME,tblName,colName,partNames);
Assert.assertNotNull(aggrStatsCached);
ColumnStatisticsObj aggrColStatsCached=aggrStatsCached.getColStats();
Assert.assertEquals(aggrColStats,aggrColStatsCached);
aggrStatsCached=cache.get("dbNotThere",tblName,colName,partNames);
Assert.assertNull(aggrStatsCached);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier partNames=preparePartNames(tables.get(0),1,9);
BloomFilter bloomFilter=prepareBloomFilter(partNames);
String tblName=tables.get(0);
String colName=tabCols.get(0);
int highVal=100, lowVal=10, numDVs=50, numNulls=5;
ColumnStatisticsObj aggrColStats=getDummyLongColStat(colName,highVal,lowVal,numDVs,numNulls);
cache.add(DB_NAME,tblName,colName,10,aggrColStats,bloomFilter);
partNames=preparePartNames(tables.get(0),1,5);
AggrColStats aggrStatsCached=cache.get(DB_NAME,tblName,colName,partNames);
Assert.assertNull(aggrStatsCached);
partNames=preparePartNames(tables.get(0),11,20);
aggrStatsCached=cache.get(DB_NAME,tblName,colName,partNames);
Assert.assertNull(aggrStatsCached);
partNames=preparePartNames(tables.get(0),1,8);
aggrStatsCached=cache.get(DB_NAME,tblName,colName,partNames);
Assert.assertNotNull(aggrStatsCached);
ColumnStatisticsObj aggrColStatsCached=aggrStatsCached.getColStats();
Assert.assertEquals(aggrColStats,aggrColStatsCached);
}
EqualityVerifier Class: org.apache.hadoop.hive.metastore.TestFilterHooks UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.metastore.TestHiveMetaStore InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier tableNames=Arrays.asList("table1","table2","table3","table4","table5");
silentDropDatabase(dbName);
Database db=new Database();
db.setName(dbName);
client.createDatabase(db);
for ( String tableName : tableNames) {
createTable(dbName,tableName);
}
List tableObjs=client.getTableObjectsByName(dbName,tableNames);
assertEquals(tableNames.size(),tableObjs.size());
for ( Table table : tableObjs) {
assertTrue(tableNames.contains(table.getTableName().toLowerCase()));
}
client.dropDatabase(dbName,true,true,true);
}
Class: org.apache.hadoop.hive.metastore.TestHiveMetaStorePartitionSpecs IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier fields=table.getSd().getCols();
fields.add(new FieldSchema("goo","string","Entirely new column. Doesn't apply to older partitions."));
table.getSd().setCols(fields);
hmsc.alter_table(dbName,tableName,table);
table=hmsc.getTable(dbName,tableName);
Assert.assertEquals("Unexpected number of table columns.",3,table.getSd().getColsSize());
populatePartitions(hmsc,table,Arrays.asList("hasNewColumn"));
PartitionSpecProxy partitionSpecProxy=hmsc.listPartitionSpecs(dbName,tableName,-1);
Assert.assertEquals("Unexpected number of partitions.",nDates * 3,partitionSpecProxy.size());
Assert.assertTrue("Unexpected type of PartitionSpecProxy.",partitionSpecProxy instanceof CompositePartitionSpecProxy);
CompositePartitionSpecProxy compositePartitionSpecProxy=(CompositePartitionSpecProxy)partitionSpecProxy;
List partitionSpecs=compositePartitionSpecProxy.toPartitionSpec();
Assert.assertTrue("PartitionSpec[0] should have been a SharedSDPartitionSpec.",partitionSpecs.get(0).isSetSharedSDPartitionSpec());
Assert.assertEquals("PartitionSpec[0] should use the table-path as the common root location. ",table.getSd().getLocation(),partitionSpecs.get(0).getRootPath());
Assert.assertTrue("PartitionSpec[1] should have been a SharedSDPartitionSpec.",partitionSpecs.get(1).isSetSharedSDPartitionSpec());
Assert.assertEquals("PartitionSpec[1] should use the table-path as the common root location. ",table.getSd().getLocation(),partitionSpecs.get(1).getRootPath());
Assert.assertTrue("PartitionSpec[2] should have been a ListComposingPartitionSpec.",partitionSpecs.get(2).isSetPartitionList());
PartitionSpecProxy.PartitionIterator iterator=partitionSpecProxy.getPartitionIterator();
Map> blurbToPartitionList=new HashMap>(3);
while (iterator.hasNext()) {
Partition partition=iterator.next();
String blurb=partition.getValues().get(1);
if (!blurbToPartitionList.containsKey(blurb)) {
blurbToPartitionList.put(blurb,new ArrayList(nDates));
}
blurbToPartitionList.get(blurb).add(partition);
}
for ( Partition partition : blurbToPartitionList.get("isLocatedOutsideTablePath")) {
Assert.assertEquals("Unexpected number of columns.",2,partition.getSd().getCols().size());
Assert.assertEquals("Unexpected first column.","foo",partition.getSd().getCols().get(0).getName());
Assert.assertEquals("Unexpected second column.","bar",partition.getSd().getCols().get(1).getName());
String partitionLocation=partition.getSd().getLocation();
String tableLocation=table.getSd().getLocation();
Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". "+ "Partition should have been outside table location: "+ tableLocation,!partitionLocation.startsWith(tableLocation));
}
for ( Partition partition : blurbToPartitionList.get("isLocatedInTablePath")) {
Assert.assertEquals("Unexpected number of columns.",2,partition.getSd().getCols().size());
Assert.assertEquals("Unexpected first column.","foo",partition.getSd().getCols().get(0).getName());
Assert.assertEquals("Unexpected second column.","bar",partition.getSd().getCols().get(1).getName());
String partitionLocation=partition.getSd().getLocation();
String tableLocation=table.getSd().getLocation();
Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". "+ "Partition should have been within table location: "+ tableLocation,partitionLocation.startsWith(tableLocation));
}
for ( Partition partition : blurbToPartitionList.get("hasNewColumn")) {
Assert.assertEquals("Unexpected number of columns.",3,partition.getSd().getCols().size());
Assert.assertEquals("Unexpected first column.","foo",partition.getSd().getCols().get(0).getName());
Assert.assertEquals("Unexpected second column.","bar",partition.getSd().getCols().get(1).getName());
Assert.assertEquals("Unexpected third column.","goo",partition.getSd().getCols().get(2).getName());
String partitionLocation=partition.getSd().getLocation();
String tableLocation=table.getSd().getLocation();
Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". "+ "Partition should have been within table location: "+ tableLocation,partitionLocation.startsWith(tableLocation));
}
}
catch ( Throwable t) {
LOG.error("Unexpected Exception!",t);
t.printStackTrace();
Assert.assertTrue("Unexpected Exception!",false);
}
}
Class: org.apache.hadoop.hive.metastore.TestHiveMetaStoreTxns BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier tids=client.openTxns("me",3).getTxn_ids();
Assert.assertEquals(1L,(long)tids.get(0));
Assert.assertEquals(2L,(long)tids.get(1));
Assert.assertEquals(3L,(long)tids.get(2));
client.rollbackTxn(1);
client.commitTxn(2);
ValidTxnList validTxns=client.getValidTxns();
Assert.assertFalse(validTxns.isTxnValid(1));
Assert.assertTrue(validTxns.isTxnValid(2));
Assert.assertFalse(validTxns.isTxnValid(3));
Assert.assertFalse(validTxns.isTxnValid(4));
}
InternalCallVerifier EqualityVerifier tids=client.openTxns("me",5).getTxn_ids();
HeartbeatTxnRangeResponse rsp=client.heartbeatTxnRange(1,5);
Assert.assertEquals(0,rsp.getNosuch().size());
Assert.assertEquals(0,rsp.getAborted().size());
client.rollbackTxn(1L);
client.commitTxn(2L);
client.commitTxn(3L);
client.commitTxn(4L);
validTxns=client.getValidTxns();
System.out.println("validTxns = " + validTxns);
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(2L,2L));
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(2L,3L));
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(2L,4L));
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(3L,4L));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(1L,4L));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(2L,5L));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(1L,2L));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(4L,5L));
Assert.assertEquals(ValidTxnList.RangeResponse.NONE,validTxns.isTxnRangeValid(1L,1L));
Assert.assertEquals(ValidTxnList.RangeResponse.NONE,validTxns.isTxnRangeValid(5L,10L));
validTxns=new ValidReadTxnList("10:4:5:6");
Assert.assertEquals(ValidTxnList.RangeResponse.NONE,validTxns.isTxnRangeValid(4,6));
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(7,10));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(7,11));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(3,6));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(4,7));
Assert.assertEquals(ValidTxnList.RangeResponse.SOME,validTxns.isTxnRangeValid(1,12));
Assert.assertEquals(ValidTxnList.RangeResponse.ALL,validTxns.isTxnRangeValid(1,3));
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier tids=client.openTxns("me",3).getTxn_ids();
Assert.assertEquals(1L,(long)tids.get(0));
Assert.assertEquals(2L,(long)tids.get(1));
Assert.assertEquals(3L,(long)tids.get(2));
client.rollbackTxn(1);
client.commitTxn(2);
ValidTxnList validTxns=client.getValidTxns(3);
Assert.assertFalse(validTxns.isTxnValid(1));
Assert.assertTrue(validTxns.isTxnValid(2));
Assert.assertTrue(validTxns.isTxnValid(3));
Assert.assertFalse(validTxns.isTxnValid(4));
}
InternalCallVerifier EqualityVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.metastore.TestLockRequestBuilder InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_READ,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(2,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(2,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(2,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
BooleanVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_READ,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(2,locks.size());
Assert.assertEquals("fred",req.getUser());
Assert.assertEquals(InetAddress.getLocalHost().getHostName(),req.getHostname());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(2,locks.size());
Assert.assertEquals("unknown",req.getUser());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_WRITE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.EXCLUSIVE,locks.get(0).getType());
}
InternalCallVerifier EqualityVerifier locks=req.getComponent();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(LockType.SHARED_READ,locks.get(0).getType());
}
Class: org.apache.hadoop.hive.metastore.TestObjectStore InternalCallVerifier EqualityVerifier tableParams=new HashMap();
tableParams.put("EXTERNAL","false");
FieldSchema partitionKey1=new FieldSchema("Country",serdeConstants.STRING_TYPE_NAME,"");
FieldSchema partitionKey2=new FieldSchema("State",serdeConstants.STRING_TYPE_NAME,"");
Table tbl1=new Table(TABLE1,DB1,"owner",1,2,3,sd,Arrays.asList(partitionKey1,partitionKey2),tableParams,"viewOriginalText","viewExpandedText","MANAGED_TABLE");
objectStore.createTable(tbl1);
HashMap partitionParams=new HashMap();
partitionParams.put("PARTITION_LEVEL_PRIVILEGE","true");
List value1=Arrays.asList("US","CA");
Partition part1=new Partition(value1,DB1,TABLE1,111,111,sd,partitionParams);
objectStore.addPartition(part1);
List value2=Arrays.asList("US","MA");
Partition part2=new Partition(value2,DB1,TABLE1,222,222,sd,partitionParams);
objectStore.addPartition(part2);
Deadline.startTimer("getPartition");
List partitions=objectStore.getPartitions(DB1,TABLE1,10);
Assert.assertEquals(2,partitions.size());
Assert.assertEquals(111,partitions.get(0).getCreateTime());
Assert.assertEquals(222,partitions.get(1).getCreateTime());
int numPartitions=objectStore.getNumPartitionsByFilter(DB1,TABLE1,"");
Assert.assertEquals(partitions.size(),numPartitions);
numPartitions=objectStore.getNumPartitionsByFilter(DB1,TABLE1,"country = \"US\"");
Assert.assertEquals(2,numPartitions);
objectStore.dropPartition(DB1,TABLE1,value1);
partitions=objectStore.getPartitions(DB1,TABLE1,10);
Assert.assertEquals(1,partitions.size());
Assert.assertEquals(222,partitions.get(0).getCreateTime());
objectStore.dropPartition(DB1,TABLE1,value2);
objectStore.dropTable(DB1,TABLE1);
objectStore.dropDatabase(DB1);
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier roles=objectStore.listRoleNames();
Assert.assertEquals(2,roles.size());
Assert.assertEquals(ROLE2,roles.get(1));
Role role1=objectStore.getRole(ROLE1);
Assert.assertEquals(OWNER,role1.getOwnerName());
objectStore.grantRole(role1,USER1,PrincipalType.USER,OWNER,PrincipalType.ROLE,true);
objectStore.revokeRole(role1,USER1,PrincipalType.USER,false);
objectStore.removeRole(ROLE1);
}
InternalCallVerifier EqualityVerifier params=new HashMap();
params.put("EXTERNAL","false");
Table tbl1=new Table(TABLE1,DB1,"owner",1,2,3,sd,null,params,"viewOriginalText","viewExpandedText","MANAGED_TABLE");
objectStore.createTable(tbl1);
List tables=objectStore.getAllTables(DB1);
Assert.assertEquals(1,tables.size());
Assert.assertEquals(TABLE1,tables.get(0));
Table newTbl1=new Table("new" + TABLE1,DB1,"owner",1,2,3,sd,null,params,"viewOriginalText","viewExpandedText","MANAGED_TABLE");
objectStore.alterTable(DB1,TABLE1,newTbl1);
tables=objectStore.getTables(DB1,"new*");
Assert.assertEquals(1,tables.size());
Assert.assertEquals("new" + TABLE1,tables.get(0));
objectStore.dropTable(DB1,"new" + TABLE1);
tables=objectStore.getAllTables(DB1);
Assert.assertEquals(0,tables.size());
objectStore.dropDatabase(DB1);
}
InternalCallVerifier EqualityVerifier databases=objectStore.getAllDatabases();
Assert.assertEquals(2,databases.size());
Assert.assertEquals(DB1,databases.get(0));
Assert.assertEquals(DB2,databases.get(1));
objectStore.dropDatabase(DB1);
databases=objectStore.getAllDatabases();
Assert.assertEquals(1,databases.size());
Assert.assertEquals(DB2,databases.get(0));
objectStore.dropDatabase(DB2);
}
Class: org.apache.hadoop.hive.metastore.TestServerSpecificConfig EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseAggrStatsCacheIntegration InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
cols.add(new FieldSchema("col2","varchar","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
for ( List partVals : Arrays.asList(partVals1,partVals2)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
obj=new ColumnStatisticsObj();
obj.setColName("col2");
obj.setColType("varchar");
data=new ColumnStatisticsData();
StringColumnStatsData scsd=new StringColumnStatsData();
scsd.setAvgColLen(10.3);
scsd.setMaxColLen(2000);
scsd.setNumNulls(3);
scsd.setNumDVs(12342);
data.setStringStats(scsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
}
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(2,aggrStats.getPartsFound());
Assert.assertEquals(2,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("boolean",cso.getColType());
BooleanColumnStatsData bcsd=cso.getStatsData().getBooleanStats();
Assert.assertEquals(20,bcsd.getNumFalses());
Assert.assertEquals(40,bcsd.getNumTrues());
Assert.assertEquals(60,bcsd.getNumNulls());
cso=aggrStats.getColStats().get(1);
Assert.assertEquals("col2",cso.getColName());
Assert.assertEquals("varchar",cso.getColType());
StringColumnStatsData scsd=cso.getStatsData().getStringStats();
Assert.assertEquals(10.3,scsd.getAvgColLen(),0.1);
Assert.assertEquals(2000,scsd.getMaxColLen());
Assert.assertEquals(6,scsd.getNumNulls());
Assert.assertEquals(12342,scsd.getNumDVs());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1","col2"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1","col2"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
store.backdoor().getStatsCache().flushMemory();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1","col2"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(2,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(6,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
}
EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
List partVals3=Arrays.asList("tomorrow");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
Partition[] partitions=new Partition[3];
int partnum=0;
for ( List partVals : Arrays.asList(partVals1,partVals2,partVals3)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
partitions[partnum++]=part;
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
}
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=tomorrow"),Arrays.asList("col1"));
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
store.backdoor().getStatsCache().wakeInvalidator();
Partition newPart=new Partition(partitions[2]);
newPart.setLastAccessTime((int)System.currentTimeMillis());
store.alterPartition(dbName,tableName,partVals3,newPart);
store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().misses.getCnt());
}
finally {
store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
store.backdoor().getStatsCache().setMaxTimeInCache(500000);
store.backdoor().getStatsCache().wakeInvalidator();
}
}
EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
List partVals3=Arrays.asList("tomorrow");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
Partition[] partitions=new Partition[3];
int partnum=0;
for ( List partVals : Arrays.asList(partVals1,partVals2,partVals3)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
partitions[partnum++]=part;
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
}
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=tomorrow"),Arrays.asList("col1"));
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
store.backdoor().getStatsCache().wakeInvalidator();
Partition[] newParts=new Partition[2];
newParts[0]=new Partition(partitions[0]);
newParts[0].setLastAccessTime((int)System.currentTimeMillis());
newParts[1]=new Partition(partitions[2]);
newParts[1].setLastAccessTime((int)System.currentTimeMillis());
store.alterPartitions(dbName,tableName,Arrays.asList(partVals1,partVals3),Arrays.asList(newParts));
store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().misses.getCnt());
}
finally {
store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
store.backdoor().getStatsCache().setMaxTimeInCache(500000);
store.backdoor().getStatsCache().wakeInvalidator();
}
}
InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","long","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
boolean first=true;
for ( List partVals : Arrays.asList(partVals1,partVals2)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/psws/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
if (first) {
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("long");
ColumnStatisticsData data=new ColumnStatisticsData();
LongColumnStatsData lcsd=new LongColumnStatsData();
lcsd.setHighValue(192L);
lcsd.setLowValue(-20L);
lcsd.setNumNulls(30);
lcsd.setNumDVs(32);
data.setLongStats(lcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
first=false;
}
}
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(1,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("long",cso.getColType());
LongColumnStatsData lcsd=cso.getStatsData().getLongStats();
Assert.assertEquals(192L,lcsd.getHighValue());
Assert.assertEquals(-20L,lcsd.getLowValue());
Assert.assertEquals(30,lcsd.getNumNulls());
Assert.assertEquals(32,lcsd.getNumDVs());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
store.backdoor().getStatsCache().flushMemory();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(1,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
}
InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
List partVals3=Arrays.asList("tomorrow");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
for ( List partVals : Arrays.asList(partVals1,partVals2,partVals3)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
}
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(2,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("boolean",cso.getColType());
BooleanColumnStatsData bcsd=cso.getStatsData().getBooleanStats();
Assert.assertEquals(20,bcsd.getNumFalses());
Assert.assertEquals(40,bcsd.getNumTrues());
Assert.assertEquals(60,bcsd.getNumNulls());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(5,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().misses.getCnt());
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals3.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(100);
bcsd.setNumTrues(200);
bcsd.setNumNulls(300);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
Checker afterUpdate=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(2,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("boolean",cso.getColType());
BooleanColumnStatsData bcsd=cso.getStatsData().getBooleanStats();
Assert.assertEquals(110,bcsd.getNumFalses());
Assert.assertEquals(220,bcsd.getNumTrues());
Assert.assertEquals(330,bcsd.getNumNulls());
}
}
;
store.updatePartitionColumnStatistics(cs,partVals3);
store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
afterUpdate.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(6,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(7,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3,store.backdoor().getStatsCache().misses.getCnt());
store.dropPartition(dbName,tableName,partVals2);
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(1,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("boolean",cso.getColType());
BooleanColumnStatsData bcsd=cso.getStatsData().getBooleanStats();
Assert.assertEquals(10,bcsd.getNumFalses());
Assert.assertEquals(20,bcsd.getNumTrues());
Assert.assertEquals(30,bcsd.getNumNulls());
}
}
.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(8,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=tomorrow","ds=today"),Arrays.asList("col1"));
afterUpdate.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(9,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(4,store.backdoor().getStatsCache().misses.getCnt());
}
finally {
store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
store.backdoor().getStatsCache().setMaxTimeInCache(500000);
store.backdoor().getStatsCache().wakeInvalidator();
}
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseAggregateStatsCache EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
for ( List partVals : Arrays.asList(partVals1,partVals2)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/nws/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
}
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(0,aggrStats.getPartsFound());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
}
InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
for ( List partVals : Arrays.asList(partVals1,partVals2)) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
Partition part=new Partition(partVals,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData bcsd=new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals);
}
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(2,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("boolean",cso.getColType());
BooleanColumnStatsData bcsd=cso.getStatsData().getBooleanStats();
Assert.assertEquals(20,bcsd.getNumFalses());
Assert.assertEquals(40,bcsd.getNumTrues());
Assert.assertEquals(60,bcsd.getNumNulls());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
}
InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(0,aggrStats.getPartsFound());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
}
InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
Partition part=new Partition(partVals1,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals1.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("double");
ColumnStatisticsData data=new ColumnStatisticsData();
DoubleColumnStatsData dcsd=new DoubleColumnStatsData();
dcsd.setHighValue(1000.2342343);
dcsd.setLowValue(-20.1234213423);
dcsd.setNumNulls(30);
dcsd.setNumDVs(12342);
data.setDoubleStats(dcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals1);
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(1,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("double",cso.getColType());
DoubleColumnStatsData dcsd=cso.getStatsData().getDoubleStats();
Assert.assertEquals(1000.23,dcsd.getHighValue(),0.01);
Assert.assertEquals(-20.12,dcsd.getLowValue(),0.01);
Assert.assertEquals(30,dcsd.getNumNulls());
Assert.assertEquals(12342,dcsd.getNumDVs());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseAggregateStatsCacheWithBitVector InternalCallVerifier EqualityVerifier partVals1=Arrays.asList("today");
List partVals2=Arrays.asList("yesterday");
long now=System.currentTimeMillis();
List cols=new ArrayList<>();
cols.add(new FieldSchema("col1","boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,Collections.emptyMap());
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbName,"me",(int)now,(int)now,0,sd,partCols,Collections.emptyMap(),null,null,null);
store.createTable(table);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
Partition part=new Partition(partVals1,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/hit/ds=" + partVals2.get(0));
part=new Partition(partVals2,dbName,tableName,(int)now,(int)now,psd,Collections.emptyMap());
store.addPartition(part);
ColumnStatistics cs=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals1.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("double");
ColumnStatisticsData data=new ColumnStatisticsData();
DoubleColumnStatsData dcsd=new DoubleColumnStatsData();
dcsd.setHighValue(1000.2342343);
dcsd.setLowValue(-20.1234213423);
dcsd.setNumNulls(30);
dcsd.setNumDVs(12342);
dcsd.setBitVectors("{0, 4, 5, 7}{0, 1}{0, 1, 2}{0, 1, 4}{0}{0, 2}{0, 3}{0, 2, 3, 4}{0, 1, 4}{0, 1}{0}{0, 1, 3, 8}{0, 2}{0, 2}{0, 9}{0, 1, 4}");
data.setDoubleStats(dcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals1);
cs=new ColumnStatistics();
desc=new ColumnStatisticsDesc(false,dbName,tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals2.get(0));
cs.setStatsDesc(desc);
obj=new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("double");
data=new ColumnStatisticsData();
dcsd=new DoubleColumnStatsData();
dcsd.setHighValue(1000.2342343);
dcsd.setLowValue(-20.1234213423);
dcsd.setNumNulls(30);
dcsd.setNumDVs(12342);
dcsd.setBitVectors("{0, 1}{0, 1}{1, 2, 4}{0, 1, 2}{0, 1, 2}{0, 2}{0, 1, 3, 4}{0, 1}{0, 1}{3, 4, 6}{2}{0, 1}{0, 3}{0}{0, 1}{0, 1, 4}");
data.setDoubleStats(dcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs,partVals2);
Checker statChecker=new Checker(){
@Override public void checkStats( AggrStats aggrStats) throws Exception {
Assert.assertEquals(2,aggrStats.getPartsFound());
Assert.assertEquals(1,aggrStats.getColStatsSize());
ColumnStatisticsObj cso=aggrStats.getColStats().get(0);
Assert.assertEquals("col1",cso.getColName());
Assert.assertEquals("double",cso.getColType());
DoubleColumnStatsData dcsd=cso.getStatsData().getDoubleStats();
Assert.assertEquals(1000.23,dcsd.getHighValue(),0.01);
Assert.assertEquals(-20.12,dcsd.getLowValue(),0.01);
Assert.assertEquals(60,dcsd.getNumNulls());
Assert.assertEquals(5,dcsd.getNumDVs());
Assert.assertEquals("{0, 1, 4, 5, 7}{0, 1}{0, 1, 2, 4}{0, 1, 2, 4}{0, 1, 2}{0, 2}{0, 1, 3, 4}{0, 1, 2, 3, 4}{0, 1, 4}{0, 1, 3, 4, 6}{0, 2}{0, 1, 3, 8}{0, 2, 3}{0, 2}{0, 1, 9}{0, 1, 4}",dcsd.getBitVectors());
}
}
;
AggrStats aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=today","ds=yesterday"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
aggrStats=store.get_aggr_stats_for(dbName,tableName,Arrays.asList("ds=yesterday","ds=today"),Arrays.asList("col1"));
statChecker.checkStats(aggrStats);
Assert.assertEquals(0,store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2,store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(1,store.backdoor().getStatsCache().misses.getCnt());
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseFilterPlanUtil EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier parts=new ArrayList();
parts.add(new FieldSchema("year","int",null));
parts.add(new FieldSchema("month","int",null));
parts.add(new FieldSchema("state","string",null));
ExpressionTree exprTree=PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
PlanResult planRes=HBaseFilterPlanUtil.getFilterPlan(exprTree,parts);
Assert.assertEquals(planRes.plan.getPlans().size(),1);
ScanPlan sp=planRes.plan.getPlans().get(0);
byte[] startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
byte[] endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
RowFilter filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,Shorts.toByteArray((short)2015)));
Assert.assertTrue(Bytes.contains(endRowSuffix,Shorts.toByteArray((short)2016)));
PartitionKeyComparator comparator=(PartitionKeyComparator)filter.getComparator();
Assert.assertEquals(comparator.ranges.size(),1);
Assert.assertEquals(comparator.ranges.get(0).keyName,"state");
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier parts=new ArrayList();
parts.add(new FieldSchema("k1","int",null));
LeafNode r=new LeafNode();
r.keyName=KEY;
r.value=VAL2;
TreeNode tn=new TreeNode(l,LogicalOperator.AND,r);
l.operator=Operator.GREATERTHANOREQUALTO;
r.operator=Operator.LESSTHAN;
verifyPlan(tn,parts,KEY,new ScanMarker(VAL1,INCLUSIVE,"int"),new ScanMarker(VAL2,!INCLUSIVE,"int"));
l.operator=Operator.GREATERTHANOREQUALTO;
r.operator=Operator.GREATERTHAN;
verifyPlan(tn,parts,KEY,new ScanMarker(VAL2,!INCLUSIVE,"int"),DEFAULT_SCANMARKER);
tn=new TreeNode(l,LogicalOperator.OR,r);
ExpressionTree e=new ExpressionTree();
e.setRootForTest(tn);
PlanResult planRes=HBaseFilterPlanUtil.getFilterPlan(e,parts);
Assert.assertEquals(2,planRes.plan.getPlans().size());
Assert.assertEquals(false,planRes.hasUnsupportedCondition);
TreeNode tn2=new TreeNode(l,LogicalOperator.AND,tn);
e=new ExpressionTree();
e.setRootForTest(tn2);
planRes=HBaseFilterPlanUtil.getFilterPlan(e,parts);
Assert.assertEquals(2,planRes.plan.getPlans().size());
Assert.assertEquals(false,planRes.hasUnsupportedCondition);
LeafNode klike=new LeafNode();
klike.keyName=KEY;
klike.value=VAL1;
klike.operator=Operator.LIKE;
TreeNode tn3=new TreeNode(tn2,LogicalOperator.OR,klike);
e=new ExpressionTree();
e.setRootForTest(tn3);
planRes=HBaseFilterPlanUtil.getFilterPlan(e,parts);
Assert.assertEquals(3,planRes.plan.getPlans().size());
Assert.assertEquals(false,planRes.hasUnsupportedCondition);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier parts=new ArrayList();
parts.add(new FieldSchema("year","string",null));
parts.add(new FieldSchema("month","string",null));
parts.add(new FieldSchema("state","string",null));
ExpressionTree exprTree=PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
PlanResult planRes=HBaseFilterPlanUtil.getFilterPlan(exprTree,parts);
Assert.assertEquals(planRes.plan.getPlans().size(),1);
ScanPlan sp=planRes.plan.getPlans().get(0);
byte[] startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
byte[] endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
RowFilter filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,"2015".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"2015".getBytes()));
Assert.assertFalse(Bytes.contains(startRowSuffix,"CA".getBytes()));
Assert.assertFalse(Bytes.contains(endRowSuffix,"CA".getBytes()));
PartitionKeyComparator comparator=(PartitionKeyComparator)filter.getComparator();
Assert.assertEquals(comparator.ranges.size(),1);
Assert.assertEquals(comparator.ranges.get(0).keyName,"state");
exprTree=PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
planRes=HBaseFilterPlanUtil.getFilterPlan(exprTree,parts);
Assert.assertEquals(planRes.plan.getPlans().size(),1);
sp=planRes.plan.getPlans().get(0);
startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,"2015".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"2015".getBytes()));
Assert.assertTrue(Bytes.contains(startRowSuffix,"10".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"11".getBytes()));
comparator=(PartitionKeyComparator)filter.getComparator();
Assert.assertEquals(comparator.ops.size(),1);
Assert.assertEquals(comparator.ops.get(0).keyName,"state");
exprTree=PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
planRes=HBaseFilterPlanUtil.getFilterPlan(exprTree,parts);
Assert.assertEquals(planRes.plan.getPlans().size(),1);
sp=planRes.plan.getPlans().get(0);
startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,"2014".getBytes()));
comparator=(PartitionKeyComparator)filter.getComparator();
Assert.assertEquals(comparator.ranges.size(),1);
Assert.assertEquals(comparator.ranges.get(0).keyName,"month");
Assert.assertEquals(comparator.ops.size(),1);
Assert.assertEquals(comparator.ops.get(0).keyName,"state");
exprTree=PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 " + "or month < 3)").tree;
planRes=HBaseFilterPlanUtil.getFilterPlan(exprTree,parts);
sp=planRes.plan.getPlans().get(0);
startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,"2014".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"2014".getBytes()));
Assert.assertTrue(Bytes.contains(startRowSuffix,"10".getBytes()));
sp=planRes.plan.getPlans().get(1);
startRowSuffix=sp.getStartRowSuffix("testdb","testtb",parts);
endRowSuffix=sp.getEndRowSuffix("testdb","testtb",parts);
filter=(RowFilter)sp.getFilter(parts);
Assert.assertTrue(Bytes.contains(startRowSuffix,"2014".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"2014".getBytes()));
Assert.assertTrue(Bytes.contains(endRowSuffix,"3".getBytes()));
}
InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseImport APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier keys=new HashSet<>(Arrays.asList(hbaseKeys));
for (int i=0; i < masterKeys.length; i++) {
Assert.assertTrue(keys.contains(masterKeys[i]));
}
}
EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier keys=new HashSet<>(Arrays.asList(hbaseKeys));
for (int i=0; i < masterKeys.length; i++) {
Assert.assertTrue(keys.contains(masterKeys[i]));
}
}
EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=new ArrayList<>();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("region","string",""));
for (int j=0; j < parallelFactor; j++) {
rdbms.createTable(new Table("t" + j,dbNames[i],"me",now,now,0,sd,partCols,emptyParameters,null,null,null));
for (int k=0; k < parallelFactor; k++) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/region=" + k);
Partition part=new Partition(Arrays.asList("p" + k),dbNames[i],"t" + j,now,now,psd,emptyParameters);
rdbms.addPartition(part);
}
}
}
HBaseImport importer=new HBaseImport("-p","2","-b","2","-d",dbNames[0]);
importer.setConnections(rdbms,store);
importer.run();
for (int i=0; i < dbNames.length; i++) {
Database db=store.getDatabase(dbNames[i]);
Assert.assertNotNull(db);
for (int j=0; j < parallelFactor; j++) {
Table table=store.getTable(db.getName(),"t" + j);
Assert.assertNotNull(table);
Assert.assertEquals(now,table.getLastAccessTime());
Assert.assertEquals("input",table.getSd().getInputFormat());
for (int k=0; k < parallelFactor; k++) {
Partition part=store.getPartition(dbNames[i],"t" + j,Arrays.asList("p" + k));
Assert.assertNotNull(part);
Assert.assertEquals("file:/tmp/region=" + k,part.getSd().getLocation());
}
Assert.assertEquals(parallelFactor,store.getPartitions(dbNames[i],"t" + j,-1).size());
}
Assert.assertEquals(parallelFactor,store.getAllTables(dbNames[i]).size());
}
}
IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=new ArrayList<>();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList<>();
partCols.add(new FieldSchema("region","string",""));
for (int j=0; j < parallelFactor; j++) {
rdbms.createTable(new Table("t" + j,dbNames[i],"me",now,now,0,sd,partCols,emptyParameters,null,null,null));
for (int k=0; k < parallelFactor; k++) {
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/region=" + k);
Partition part=new Partition(Arrays.asList("p" + k),dbNames[i],"t" + j,now,now,psd,emptyParameters);
rdbms.addPartition(part);
}
}
}
HBaseImport importer=new HBaseImport("-p","2","-b","2","-d",dbNames[0]);
importer.setConnections(rdbms,store);
importer.run();
for (int i=0; i < dbNames.length; i++) {
Database db=store.getDatabase(dbNames[i]);
Assert.assertNotNull(db);
for (int j=0; j < parallelFactor; j++) {
Table table=store.getTable(db.getName(),"t" + j);
Assert.assertNotNull(table);
Assert.assertEquals(now,table.getLastAccessTime());
Assert.assertEquals("input",table.getSd().getInputFormat());
for (int k=0; k < parallelFactor; k++) {
Partition part=store.getPartition(dbNames[i],"t" + j,Arrays.asList("p" + k));
Assert.assertNotNull(part);
Assert.assertEquals("file:/tmp/region=" + k,part.getSd().getLocation());
}
Assert.assertEquals(parallelFactor,store.getPartitions(dbNames[i],"t" + j,-1).size());
}
Assert.assertEquals(parallelFactor,store.getAllTables(dbNames[i]).size());
}
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseMetastoreSql EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseSchemaTool EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier privileges=new ArrayList<>();
HiveObjectRef hiveObjRef=new HiveObjectRef(HiveObjectType.GLOBAL,"db0","tab0",null,null);
PrivilegeGrantInfo grantInfo=new PrivilegeGrantInfo("read",0,"me",PrincipalType.USER,false);
HiveObjectPrivilege hop=new HiveObjectPrivilege(hiveObjRef,"user",PrincipalType.USER,grantInfo);
privileges.add(hop);
grantInfo=new PrivilegeGrantInfo("create",0,"me",PrincipalType.USER,true);
hop=new HiveObjectPrivilege(hiveObjRef,"user",PrincipalType.USER,grantInfo);
privileges.add(hop);
PrivilegeBag pBag=new PrivilegeBag(privileges);
store.grantPrivileges(pBag);
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.GLOBAL_PRIVS_TABLE,null,null,conf,out,err);
Assert.assertEquals("{\"userPrivileges\":{\"user\":[{\"privilege\":\"read\",\"createTime\":0," + "\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":0},{\"privilege\":\"create\"," + "\"createTime\":0,\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":1}]}}"+ lsep,outStr.toString());
String[] tableNames=new String[3];
for (int i=0; i < tableNames.length; i++) {
tableNames[i]="tab" + i;
StorageDescriptor sd=new StorageDescriptor(Arrays.asList(new FieldSchema("col1","int",""),new FieldSchema("col2","varchar(32)","")),"/tmp",null,null,false,0,null,null,null,Collections.emptyMap());
Table tab=new Table(tableNames[i],dbNames[0],"me",0,0,0,sd,Arrays.asList(new FieldSchema("pcol1","string",""),new FieldSchema("pcol2","string","")),Collections.emptyMap(),null,null,null);
store.createTable(tab);
}
ColumnStatisticsDesc tableStatsDesc=new ColumnStatisticsDesc(false,"db0","tab0");
ColumnStatisticsData tcsd=new ColumnStatisticsData();
LongColumnStatsData tlcsd=new LongColumnStatsData(1,2);
tlcsd.setLowValue(-95);
tlcsd.setHighValue(95);
tcsd.setLongStats(tlcsd);
ColumnStatisticsData tcsd2=new ColumnStatisticsData();
tcsd2.setStringStats(new StringColumnStatsData(97,18.78,29,397));
List tcsos=Arrays.asList(new ColumnStatisticsObj("col1","int",tcsd),new ColumnStatisticsObj("col2","varchar(32)",tcsd2));
ColumnStatistics tStatObj=new ColumnStatistics(tableStatsDesc,tcsos);
store.updateTableColumnStatistics(tStatObj);
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.TABLE_TABLE,"db0.tab1",null,conf,out,err);
Assert.assertEquals("{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\"," + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"},"+ "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{},"+ "\"tableType\":\"\"} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats:"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.TABLE_TABLE,null,"db0.*",conf,out,err);
Assert.assertEquals("{\"tableName\":\"tab0\",\"dbName\":\"db0\",\"owner\":\"me\"," + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"},"+ "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"},"+ "\"tableType\":\"\"} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column "+ "col1: {\"colName\":\"col1\",\"colType\":\"int\","+ "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1,"+ "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\","+ "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,"+ "\"numNulls\":29,\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep + "{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0,"+ "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\","+ "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\","+ "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\"} sdHash: "+ "qQTgZAi5VzgpozzFGmIVTQ stats:"+ lsep+ "{\"tableName\":\"tab2\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0,"+ "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\","+ "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\","+ "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\"} sdHash: "+ "qQTgZAi5VzgpozzFGmIVTQ stats:"+ lsep,outStr.toString());
List> partVals=Arrays.asList(Arrays.asList("a","b"),Arrays.asList("c","d"));
for ( List pv : partVals) {
StorageDescriptor sd=new StorageDescriptor(Arrays.asList(new FieldSchema("col1","int",""),new FieldSchema("col2","varchar(32)","")),"/tmp",null,null,false,0,null,null,null,Collections.emptyMap());
Partition p=new Partition(pv,"db0","tab1",0,0,sd,Collections.emptyMap());
store.addPartition(p);
}
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.PART_TABLE,"db0.tab1.a.b",null,conf,out,err);
Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: " + "qQTgZAi5VzgpozzFGmIVTQ stats:"+ lsep,outStr.toString());
ColumnStatisticsDesc statsDesc=new ColumnStatisticsDesc(false,"db0","tab1");
statsDesc.setPartName("pcol1=c/pcol2=d");
ColumnStatisticsData csd1=new ColumnStatisticsData();
LongColumnStatsData lcsd=new LongColumnStatsData(1,2);
lcsd.setLowValue(-95);
lcsd.setHighValue(95);
csd1.setLongStats(lcsd);
ColumnStatisticsData csd2=new ColumnStatisticsData();
csd2.setStringStats(new StringColumnStatsData(97,18.78,29,397));
List csos=Arrays.asList(new ColumnStatisticsObj("col1","int",csd1),new ColumnStatisticsObj("col2","varchar(32)",csd2));
ColumnStatistics statsObj=new ColumnStatistics(statsDesc,csos);
store.updatePartitionColumnStatistics(statsObj,partVals.get(1));
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.PART_TABLE,"db0.tab1.c.d",null,conf,out,err);
Assert.assertEquals("{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " + "stats: column col1: {\"colName\":\"col1\",\"colType\":\"int\","+ "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1,"+ "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\","+ "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29,"+ "\"numDVs\":397,\"bitVectors\":\"\"}}}"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.PART_TABLE,null,"db0.tab1",conf,out,err);
Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " + "stats:" + lsep + "{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\",\"createTime\":0,"+ "\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column "+ "col1: {\"colName\":\"col1\",\"colType\":\"int\","+ "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1,"+ "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\","+ "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29,"+ "\"numDVs\":397,\"bitVectors\":\"\"}}}"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.PART_TABLE,null,"db0.tab1.a",conf,out,err);
Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " + "stats:"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.SD_TABLE,"qQTgZAi5VzgpozzFGmIVTQ",null,conf,out,err);
Assert.assertEquals("{\"cols\":[{\"name\":\"col1\",\"type\":\"int\",\"comment\":\"\"}," + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," + "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.SD_TABLE,null,"whatever",conf,out,err);
Assert.assertEquals("qQTgZAi5VzgpozzFGmIVTQ: {\"cols\":[{\"name\":\"col1\",\"type\":\"int\"," + "\"comment\":\"\"}," + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0,"+ "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.SECURITY_TABLE,null,"whatever",conf,out,err);
Assert.assertEquals("No security related entries" + lsep,outStr.toString());
store.addMasterKey("this be a key");
store.addToken("tokenid","delegation token");
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.SECURITY_TABLE,null,"whatever",conf,out,err);
Assert.assertEquals("Master key 0: this be a key" + lsep + "Delegation token tokenid: delegation token"+ lsep,outStr.toString());
outStr=new ByteArrayOutputStream();
out=new PrintStream(outStr);
tool.go(false,HBaseReadWrite.SEQUENCES_TABLE,null,"whatever",conf,out,err);
Assert.assertEquals("mk: 1" + lsep,outStr.toString());
}
EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseSchemaTool2 EqualityVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseStore APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(BINARY_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS,dataFromDB.getSetField());
BinaryColumnStatsData binaryDataFromDB=dataFromDB.getBinaryStats();
Assert.assertEquals(binaryData.getMaxColLen(),binaryDataFromDB.getMaxColLen());
Assert.assertEquals(binaryData.getAvgColLen(),binaryDataFromDB.getAvgColLen(),0.01);
Assert.assertEquals(binaryData.getNumNulls(),binaryDataFromDB.getNumNulls());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(STRING_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS,dataFromDB.getSetField());
StringColumnStatsData stringDataFromDB=dataFromDB.getStringStats();
Assert.assertEquals(stringData.getMaxColLen(),stringDataFromDB.getMaxColLen());
Assert.assertEquals(stringData.getAvgColLen(),stringDataFromDB.getAvgColLen(),0.01);
Assert.assertEquals(stringData.getNumNulls(),stringDataFromDB.getNumNulls());
Assert.assertEquals(stringData.getNumDVs(),stringDataFromDB.getNumDVs());
}
InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,DB,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
for ( String val : partVals) {
List vals=new ArrayList();
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,DB,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(DB,tableName,vals);
Assert.assertEquals("file:/tmp/pc=" + val,p.getSd().getLocation());
}
List parts=store.getPartitions(DB,tableName,-1);
Assert.assertEquals(5,parts.size());
String[] pv=new String[5];
for (int i=0; i < 5; i++) pv[i]=parts.get(i).getValues().get(0);
Arrays.sort(pv);
Assert.assertArrayEquals(pv,partVals.toArray(new String[5]));
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",true,0,serde,null,null,emptyParameters);
Map,String> map=new HashMap,String>();
map.put(Arrays.asList("col3"),"col4");
SkewedInfo skew=new SkewedInfo(Arrays.asList("col1"),Arrays.asList(Arrays.asList("col2")),map);
sd.setSkewedInfo(skew);
MessageDigest md=MessageDigest.getInstance("MD5");
byte[] baseHash=HBaseUtils.hashStorageDescriptor(sd,md);
StorageDescriptor changeSchema=new StorageDescriptor(sd);
changeSchema.getCols().add(new FieldSchema("col2","varchar(32)","a comment"));
byte[] schemaHash=HBaseUtils.hashStorageDescriptor(changeSchema,md);
Assert.assertFalse(Arrays.equals(baseHash,schemaHash));
StorageDescriptor changeLocation=new StorageDescriptor(sd);
changeLocation.setLocation("file:/somewhere/else");
byte[] locationHash=HBaseUtils.hashStorageDescriptor(changeLocation,md);
Assert.assertArrayEquals(baseHash,locationHash);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,DB,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List vals=Arrays.asList("fred");
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=fred");
Partition part=new Partition(vals,DB,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
part.setLastAccessTime(startTime + 10);
store.alterPartition(DB,tableName,vals,part);
Partition p=store.getPartition(DB,tableName,vals);
Assert.assertEquals(1,p.getSd().getColsSize());
Assert.assertEquals("col1",p.getSd().getCols().get(0).getName());
Assert.assertEquals("int",p.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",p.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",p.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",p.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp/pc=fred",p.getSd().getLocation());
Assert.assertEquals("input",p.getSd().getInputFormat());
Assert.assertEquals("output",p.getSd().getOutputFormat());
Assert.assertEquals(DB,p.getDbName());
Assert.assertEquals(tableName,p.getTableName());
Assert.assertEquals(1,p.getValuesSize());
Assert.assertEquals("fred",p.getValues().get(0));
Assert.assertEquals(startTime + 10,p.getLastAccessTime());
Assert.assertTrue(store.doesPartitionExist(DB,tableName,vals));
Assert.assertFalse(store.doesPartitionExist(DB,tableName,Arrays.asList("bob")));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(INT_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS,dataFromDB.getSetField());
LongColumnStatsData longDataFromDB=dataFromDB.getLongStats();
Assert.assertEquals(longData.getHighValue(),longDataFromDB.getHighValue());
Assert.assertEquals(longData.getLowValue(),longDataFromDB.getLowValue());
Assert.assertEquals(longData.getNumNulls(),longDataFromDB.getNumNulls());
Assert.assertEquals(longData.getNumDVs(),longDataFromDB.getNumDVs());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
Map params=new HashMap();
params.put("key","value");
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,17,serde,Arrays.asList("bucketcol"),Arrays.asList(new Order("sortcol",1)),params);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
Table t=store.getTable("default",tableName);
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertFalse(t.getSd().isCompressed());
Assert.assertEquals(17,t.getSd().getNumBuckets());
Assert.assertEquals(1,t.getSd().getBucketColsSize());
Assert.assertEquals("bucketcol",t.getSd().getBucketCols().get(0));
Assert.assertEquals(1,t.getSd().getSortColsSize());
Assert.assertEquals("sortcol",t.getSd().getSortCols().get(0).getCol());
Assert.assertEquals(1,t.getSd().getSortCols().get(0).getOrder());
Assert.assertEquals(1,t.getSd().getParametersSize());
Assert.assertEquals("value",t.getSd().getParameters().get("key"));
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
Assert.assertEquals(0,t.getParametersSize());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
partCols.add(new FieldSchema("region","string",""));
Table table=new Table(tableName,DB,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
String[][] partVals=new String[][]{{"today","north america"},{"tomorrow","europe"}};
for ( String[] pv : partVals) {
List vals=new ArrayList();
for ( String v : pv) vals.add(v);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + pv[0] + "/region="+ pv[1]);
Partition part=new Partition(vals,DB,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
}
List names=store.listPartitionNames(DB,tableName,(short)-1);
Assert.assertEquals(2,names.size());
String[] resultNames=names.toArray(new String[names.size()]);
Arrays.sort(resultNames);
Assert.assertArrayEquals(resultNames,new String[]{"pc=today/region=north america","pc=tomorrow/region=europe"});
List parts=store.getPartitionsByNames(DB,tableName,names);
Assert.assertArrayEquals(partVals[0],parts.get(0).getValues().toArray(new String[2]));
Assert.assertArrayEquals(partVals[1],parts.get(1).getValues().toArray(new String[2]));
store.dropPartitions(DB,tableName,names);
List afterDropParts=store.getPartitions(DB,tableName,-1);
Assert.assertEquals(0,afterDropParts.size());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(DECIMAL_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS,dataFromDB.getSetField());
DecimalColumnStatsData decimalDataFromDB=dataFromDB.getDecimalStats();
Assert.assertEquals(decimalData.getHighValue(),decimalDataFromDB.getHighValue());
Assert.assertEquals(decimalData.getLowValue(),decimalDataFromDB.getLowValue());
Assert.assertEquals(decimalData.getNumNulls(),decimalDataFromDB.getNumNulls());
Assert.assertEquals(decimalData.getNumDVs(),decimalDataFromDB.getNumDVs());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,DB,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List vals=Arrays.asList("fred");
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=fred");
Partition part=new Partition(vals,DB,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(DB,tableName,vals);
Assert.assertEquals(1,p.getSd().getColsSize());
Assert.assertEquals("col1",p.getSd().getCols().get(0).getName());
Assert.assertEquals("int",p.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",p.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",p.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",p.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp/pc=fred",p.getSd().getLocation());
Assert.assertEquals("input",p.getSd().getInputFormat());
Assert.assertEquals("output",p.getSd().getOutputFormat());
Assert.assertEquals(DB,p.getDbName());
Assert.assertEquals(tableName,p.getTableName());
Assert.assertEquals(1,p.getValuesSize());
Assert.assertEquals("fred",p.getValues().get(0));
Assert.assertTrue(store.doesPartitionExist(DB,tableName,vals));
Assert.assertFalse(store.doesPartitionExist(DB,tableName,Arrays.asList("bob")));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
startTime+=10;
table.setLastAccessTime(startTime);
store.alterTable("default",tableName,table);
Table t=store.getTable("default",tableName);
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
Assert.assertEquals(startTime,t.getLastAccessTime());
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",true,0,serde,null,null,emptyParameters);
Map,String> map=new HashMap,String>();
map.put(Arrays.asList("col3"),"col4");
SkewedInfo skew=new SkewedInfo(Arrays.asList("col1"),Arrays.asList(Arrays.asList("col2")),map);
sd.setSkewedInfo(skew);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
Table t=store.getTable("default",tableName);
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertTrue(t.getSd().isCompressed());
Assert.assertEquals(0,t.getSd().getNumBuckets());
Assert.assertEquals(0,t.getSd().getSortColsSize());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
Assert.assertEquals(0,t.getParametersSize());
skew=t.getSd().getSkewedInfo();
Assert.assertNotNull(skew);
Assert.assertEquals(1,skew.getSkewedColNamesSize());
Assert.assertEquals("col1",skew.getSkewedColNames().get(0));
Assert.assertEquals(1,skew.getSkewedColValuesSize());
Assert.assertEquals("col2",skew.getSkewedColValues().get(0).get(0));
Assert.assertEquals(1,skew.getSkewedColValueLocationMapsSize());
Assert.assertEquals("col4",skew.getSkewedColValueLocationMaps().get(Arrays.asList("col3")));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(DOUBLE_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS,dataFromDB.getSetField());
DoubleColumnStatsData doubleDataFromDB=dataFromDB.getDoubleStats();
Assert.assertEquals(doubleData.getHighValue(),doubleDataFromDB.getHighValue(),0.01);
Assert.assertEquals(doubleData.getLowValue(),doubleDataFromDB.getLowValue(),0.01);
Assert.assertEquals(doubleData.getNumNulls(),doubleDataFromDB.getNumNulls());
Assert.assertEquals(doubleData.getNumDVs(),doubleDataFromDB.getNumDVs());
}
InternalCallVerifier EqualityVerifier uris=new ArrayList();
uris.add(new ResourceUri(ResourceType.FILE,"whatever"));
Function func=new Function(funcName,DB,"o.a.h.h.myfunc","me",PrincipalType.USER,now,FunctionType.JAVA,uris);
store.createFunction(func);
Function f=store.getFunction(DB,funcName);
Assert.assertEquals(ResourceType.FILE,f.getResourceUris().get(0).getResourceType());
func.addToResourceUris(new ResourceUri(ResourceType.ARCHIVE,"file"));
store.alterFunction(DB,funcName,func);
f=store.getFunction(DB,funcName);
Assert.assertEquals(2,f.getResourceUrisSize());
Assert.assertEquals(ResourceType.FILE,f.getResourceUris().get(0).getResourceType());
Assert.assertEquals(ResourceType.ARCHIVE,f.getResourceUris().get(1).getResourceType());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(BOOLEAN_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS,dataFromDB.getSetField());
BooleanColumnStatsData boolDataFromDB=dataFromDB.getBooleanStats();
Assert.assertEquals(boolData.getNumTrues(),boolDataFromDB.getNumTrues());
Assert.assertEquals(boolData.getNumFalses(),boolDataFromDB.getNumFalses());
Assert.assertEquals(boolData.getNumNulls(),boolDataFromDB.getNumNulls());
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseStoreBitVector APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(DECIMAL_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS,dataFromDB.getSetField());
DecimalColumnStatsData decimalDataFromDB=dataFromDB.getDecimalStats();
Assert.assertEquals(decimalData.getHighValue(),decimalDataFromDB.getHighValue());
Assert.assertEquals(decimalData.getLowValue(),decimalDataFromDB.getLowValue());
Assert.assertEquals(decimalData.getNumNulls(),decimalDataFromDB.getNumNulls());
Assert.assertEquals(decimalData.getNumDVs(),decimalDataFromDB.getNumDVs());
Assert.assertEquals(decimalData.getBitVectors(),decimalDataFromDB.getBitVectors());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(INT_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS,dataFromDB.getSetField());
LongColumnStatsData longDataFromDB=dataFromDB.getLongStats();
Assert.assertEquals(longData.getHighValue(),longDataFromDB.getHighValue());
Assert.assertEquals(longData.getLowValue(),longDataFromDB.getLowValue());
Assert.assertEquals(longData.getNumNulls(),longDataFromDB.getNumNulls());
Assert.assertEquals(longData.getNumDVs(),longDataFromDB.getNumDVs());
Assert.assertEquals(longData.getBitVectors(),longDataFromDB.getBitVectors());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(DOUBLE_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS,dataFromDB.getSetField());
DoubleColumnStatsData doubleDataFromDB=dataFromDB.getDoubleStats();
Assert.assertEquals(doubleData.getHighValue(),doubleDataFromDB.getHighValue(),0.01);
Assert.assertEquals(doubleData.getLowValue(),doubleDataFromDB.getLowValue(),0.01);
Assert.assertEquals(doubleData.getNumNulls(),doubleDataFromDB.getNumNulls());
Assert.assertEquals(doubleData.getNumDVs(),doubleDataFromDB.getNumDVs());
Assert.assertEquals(doubleData.getBitVectors(),doubleDataFromDB.getBitVectors());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier parVals=new ArrayList();
parVals.add(STRING_VAL);
store.updatePartitionColumnStatistics(stats,parVals);
List partNames=new ArrayList();
partNames.add(desc.getPartName());
List colNames=new ArrayList();
colNames.add(obj.getColName());
List statsFromDB=store.getPartitionColumnStatistics(DB,TBL,partNames,colNames);
Assert.assertEquals(1,statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(),statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB,statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL,statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB=statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB=objFromDB.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS,dataFromDB.getSetField());
StringColumnStatsData stringDataFromDB=dataFromDB.getStringStats();
Assert.assertEquals(stringData.getMaxColLen(),stringDataFromDB.getMaxColLen());
Assert.assertEquals(stringData.getAvgColLen(),stringDataFromDB.getAvgColLen(),0.01);
Assert.assertEquals(stringData.getNumNulls(),stringDataFromDB.getNumNulls());
Assert.assertEquals(stringData.getNumDVs(),stringDataFromDB.getNumDVs());
Assert.assertEquals(stringData.getBitVectors(),stringDataFromDB.getBitVectors());
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseStoreCached InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List vals=Arrays.asList("fred");
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=fred");
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(dbName,tableName,vals);
Assert.assertEquals(1,p.getSd().getColsSize());
Assert.assertEquals("col1",p.getSd().getCols().get(0).getName());
Assert.assertEquals("int",p.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",p.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",p.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",p.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp/pc=fred",p.getSd().getLocation());
Assert.assertEquals("input",p.getSd().getInputFormat());
Assert.assertEquals("output",p.getSd().getOutputFormat());
Assert.assertEquals(dbName,p.getDbName());
Assert.assertEquals(tableName,p.getTableName());
Assert.assertEquals(1,p.getValuesSize());
Assert.assertEquals("fred",p.getValues().get(0));
}
InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
Table t=store.getTable("default",tableName);
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
partCols.add(new FieldSchema("region","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
String[][] partVals=new String[][]{{"today","north america"},{"tomorrow","europe"}};
for ( String[] pv : partVals) {
List vals=new ArrayList();
for ( String v : pv) vals.add(v);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + pv[0] + "/region="+ pv[1]);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
}
List names=store.listPartitionNames(dbName,tableName,(short)-1);
Assert.assertEquals(2,names.size());
String[] resultNames=names.toArray(new String[names.size()]);
Arrays.sort(resultNames);
Assert.assertArrayEquals(resultNames,new String[]{"pc=today/region=north america","pc=tomorrow/region=europe"});
List parts=store.getPartitionsByNames(dbName,tableName,names);
Assert.assertArrayEquals(partVals[0],parts.get(0).getValues().toArray(new String[2]));
Assert.assertArrayEquals(partVals[1],parts.get(1).getValues().toArray(new String[2]));
store.dropPartitions(dbName,tableName,names);
List afterDropParts=store.getPartitions(dbName,tableName,-1);
Assert.assertEquals(0,afterDropParts.size());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema(boolcol,"boolean","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,dbname,"me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
long trues=37;
long falses=12;
long booleanNulls=2;
ColumnStatistics stats=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(true);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName(boolcol);
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData boolData=new BooleanColumnStatsData();
boolData.setNumTrues(trues);
boolData.setNumFalses(falses);
boolData.setNumNulls(booleanNulls);
data.setBooleanStats(boolData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updateTableColumnStatistics(stats);
stats=store.getTableColumnStatistics(dbname,tableName,Arrays.asList(boolcol));
Assert.assertEquals(now,stats.getStatsDesc().getLastAnalyzed());
Assert.assertEquals(dbname,stats.getStatsDesc().getDbName());
Assert.assertEquals(tableName,stats.getStatsDesc().getTableName());
Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
Assert.assertEquals(1,stats.getStatsObjSize());
ColumnStatisticsData colData=obj.getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS,colData.getSetField());
boolData=colData.getBooleanStats();
Assert.assertEquals(trues,boolData.getNumTrues());
Assert.assertEquals(falses,boolData.getNumFalses());
Assert.assertEquals(booleanNulls,boolData.getNumNulls());
}
InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
startTime+=10;
table.setLastAccessTime(startTime);
store.alterTable("default",tableName,table);
Table t=store.getTable("default",tableName);
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
Assert.assertEquals(startTime,t.getLastAccessTime());
}
InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
for ( String val : partVals) {
List vals=new ArrayList();
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(dbName,tableName,vals);
Assert.assertEquals("file:/tmp/pc=" + val,p.getSd().getLocation());
}
List parts=store.getPartitions(dbName,tableName,-1);
Assert.assertEquals(5,parts.size());
String[] pv=new String[5];
for (int i=0; i < 5; i++) pv[i]=parts.get(i).getValues().get(0);
Arrays.sort(pv);
Assert.assertArrayEquals(pv,partVals.toArray(new String[5]));
}
Class: org.apache.hadoop.hive.metastore.hbase.TestHBaseStoreIntegration APIUtilityVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
for ( String val : partVals) {
List vals=new ArrayList();
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(dbName,tableName,vals);
Assert.assertEquals("file:/tmp/pc=" + val,p.getSd().getLocation());
}
List parts=store.getPartitions(dbName,tableName,-1);
Assert.assertEquals(5,parts.size());
String[] pv=new String[5];
for (int i=0; i < 5; i++) pv[i]=parts.get(i).getValues().get(0);
Arrays.sort(pv);
Assert.assertArrayEquals(pv,partVals.toArray(new String[5]));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema(boolcol,"boolean","nocomment"));
cols.add(new FieldSchema(longcol,"long","nocomment"));
cols.add(new FieldSchema(doublecol,"double","nocomment"));
cols.add(new FieldSchema(stringcol,"varchar(32)","nocomment"));
cols.add(new FieldSchema(binarycol,"binary","nocomment"));
cols.add(new FieldSchema(decimalcol,"decimal(5, 3)","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,dbname,"me",(int)now / 1000,(int)now / 1000,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
ColumnStatistics stats=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(true);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName(boolcol);
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData boolData=new BooleanColumnStatsData();
boolData.setNumTrues(trues);
boolData.setNumFalses(falses);
boolData.setNumNulls(booleanNulls);
data.setBooleanStats(boolData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
obj=new ColumnStatisticsObj();
obj.setColName(longcol);
obj.setColType("long");
data=new ColumnStatisticsData();
LongColumnStatsData longData=new LongColumnStatsData();
longData.setHighValue(longHigh);
longData.setLowValue(longLow);
longData.setNumNulls(longNulls);
longData.setNumDVs(longDVs);
data.setLongStats(longData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
obj=new ColumnStatisticsObj();
obj.setColName(doublecol);
obj.setColType("double");
data=new ColumnStatisticsData();
DoubleColumnStatsData doubleData=new DoubleColumnStatsData();
doubleData.setHighValue(doubleHigh);
doubleData.setLowValue(doubleLow);
doubleData.setNumNulls(doubleNulls);
doubleData.setNumDVs(doubleDVs);
data.setDoubleStats(doubleData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updateTableColumnStatistics(stats);
stats=store.getTableColumnStatistics(dbname,tableName,Arrays.asList(boolcol,longcol,doublecol));
Assert.assertEquals(3,stats.getStatsObjSize());
stats=store.getTableColumnStatistics(dbname,tableName,Arrays.asList(boolcol));
Assert.assertEquals(1,stats.getStatsObjSize());
stats=new ColumnStatistics();
stats.setStatsDesc(desc);
obj=new ColumnStatisticsObj();
obj.setColName(stringcol);
obj.setColType("string");
data=new ColumnStatisticsData();
StringColumnStatsData strData=new StringColumnStatsData();
strData.setMaxColLen(strMaxLen);
strData.setAvgColLen(strAvgLen);
strData.setNumNulls(strNulls);
strData.setNumDVs(strDVs);
data.setStringStats(strData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
obj=new ColumnStatisticsObj();
obj.setColName(binarycol);
obj.setColType("binary");
data=new ColumnStatisticsData();
BinaryColumnStatsData binData=new BinaryColumnStatsData();
binData.setMaxColLen(binMaxLen);
binData.setAvgColLen(binAvgLen);
binData.setNumNulls(binNulls);
data.setBinaryStats(binData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
obj=new ColumnStatisticsObj();
obj.setColName(decimalcol);
obj.setColType("decimal(5,3)");
data=new ColumnStatisticsData();
DecimalColumnStatsData decData=new DecimalColumnStatsData();
LOG.debug("Setting decimal high value to " + decHigh.getScale() + " <"+ new String(decHigh.getUnscaled())+ ">");
decData.setHighValue(decHigh);
decData.setLowValue(decLow);
decData.setNumNulls(decNulls);
decData.setNumDVs(decDVs);
data.setDecimalStats(decData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updateTableColumnStatistics(stats);
stats=store.getTableColumnStatistics(dbname,tableName,Arrays.asList(boolcol,longcol,doublecol,stringcol,binarycol,decimalcol));
Assert.assertEquals(now,stats.getStatsDesc().getLastAnalyzed());
Assert.assertEquals(dbname,stats.getStatsDesc().getDbName());
Assert.assertEquals(tableName,stats.getStatsDesc().getTableName());
Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
Assert.assertEquals(6,stats.getStatsObjSize());
ColumnStatisticsData colData=stats.getStatsObj().get(0).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS,colData.getSetField());
boolData=colData.getBooleanStats();
Assert.assertEquals(trues,boolData.getNumTrues());
Assert.assertEquals(falses,boolData.getNumFalses());
Assert.assertEquals(booleanNulls,boolData.getNumNulls());
colData=stats.getStatsObj().get(1).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS,colData.getSetField());
longData=colData.getLongStats();
Assert.assertEquals(longHigh,longData.getHighValue());
Assert.assertEquals(longLow,longData.getLowValue());
Assert.assertEquals(longNulls,longData.getNumNulls());
Assert.assertEquals(longDVs,longData.getNumDVs());
colData=stats.getStatsObj().get(2).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS,colData.getSetField());
doubleData=colData.getDoubleStats();
Assert.assertEquals(doubleHigh,doubleData.getHighValue(),0.01);
Assert.assertEquals(doubleLow,doubleData.getLowValue(),0.01);
Assert.assertEquals(doubleNulls,doubleData.getNumNulls());
Assert.assertEquals(doubleDVs,doubleData.getNumDVs());
colData=stats.getStatsObj().get(3).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS,colData.getSetField());
strData=colData.getStringStats();
Assert.assertEquals(strMaxLen,strData.getMaxColLen());
Assert.assertEquals(strAvgLen,strData.getAvgColLen(),0.01);
Assert.assertEquals(strNulls,strData.getNumNulls());
Assert.assertEquals(strDVs,strData.getNumDVs());
colData=stats.getStatsObj().get(4).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS,colData.getSetField());
binData=colData.getBinaryStats();
Assert.assertEquals(binMaxLen,binData.getMaxColLen());
Assert.assertEquals(binAvgLen,binData.getAvgColLen(),0.01);
Assert.assertEquals(binNulls,binData.getNumNulls());
colData=stats.getStatsObj().get(5).getStatsData();
Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS,colData.getSetField());
decData=colData.getDecimalStats();
Assert.assertEquals(decHigh,decData.getHighValue());
Assert.assertEquals(decLow,decData.getLowValue());
Assert.assertEquals(decNulls,decData.getNumNulls());
Assert.assertEquals(decDVs,decData.getNumDVs());
}
APIUtilityVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
partCols.add(new FieldSchema("region","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
String[][] partVals=new String[][]{{"today","north america"},{"today","europe"},{"tomorrow","north america"},{"tomorrow","europe"}};
for ( String[] pv : partVals) {
List vals=new ArrayList();
for ( String v : pv) vals.add(v);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/ds=" + pv[0] + "/region="+ pv[1]);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
}
List partitionNames=store.listPartitionNamesPs(dbName,tableName,Arrays.asList(partVals[0]),(short)-1);
Assert.assertEquals(1,partitionNames.size());
Assert.assertEquals("ds=today/region=north america",partitionNames.get(0));
partitionNames=store.listPartitionNamesPs(dbName,tableName,Arrays.asList(partVals[0][0]),(short)-1);
Assert.assertEquals(2,partitionNames.size());
String[] names=partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[]{"ds=today/region=europe","ds=today/region=north america"},names);
partitionNames=store.listPartitionNamesPs(dbName,tableName,Arrays.asList("today","*"),(short)-1);
Assert.assertEquals(2,partitionNames.size());
names=partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[]{"ds=today/region=europe","ds=today/region=north america"},names);
partitionNames=store.listPartitionNamesPs(dbName,tableName,Arrays.asList("*","europe"),(short)-1);
Assert.assertEquals(2,partitionNames.size());
names=partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[]{"ds=today/region=europe","ds=tomorrow/region=europe"},names);
}
BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier privileges=new ArrayList();
HiveObjectRef hiveObjRef=new HiveObjectRef(HiveObjectType.GLOBAL,null,null,null,null);
PrivilegeGrantInfo grantInfo=new PrivilegeGrantInfo("read",now,"me",PrincipalType.USER,false);
HiveObjectPrivilege hop=new HiveObjectPrivilege(hiveObjRef,userNames[0],PrincipalType.USER,grantInfo);
privileges.add(hop);
grantInfo=new PrivilegeGrantInfo("write",now,"me",PrincipalType.USER,true);
hop=new HiveObjectPrivilege(hiveObjRef,roleNames[0],PrincipalType.ROLE,grantInfo);
privileges.add(hop);
PrivilegeBag pBag=new PrivilegeBag(privileges);
store.grantPrivileges(pBag);
List hops=store.listPrincipalGlobalGrants(roleNames[0],PrincipalType.ROLE);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.ROLE,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.GLOBAL,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("write",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalGlobalGrants(userNames[0],PrincipalType.USER);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.USER,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.GLOBAL,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("read",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalGlobalGrants(roleNames[1],PrincipalType.ROLE);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalGlobalGrants(userNames[1],PrincipalType.USER);
Assert.assertEquals(0,hops.size());
hops=store.listGlobalGrantsAll();
Assert.assertEquals(2,hops.size());
boolean sawUser=false, sawRole=false;
for ( HiveObjectPrivilege h : hops) {
if (h.getPrincipalName().equals(userNames[0])) {
Assert.assertEquals(PrincipalType.USER,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.GLOBAL,h.getHiveObject().getObjectType());
Assert.assertEquals("read",h.getGrantInfo().getPrivilege());
sawUser=true;
}
else if (h.getPrincipalName().equals(roleNames[0])) {
Assert.assertEquals(PrincipalType.ROLE,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.GLOBAL,h.getHiveObject().getObjectType());
Assert.assertEquals("write",h.getGrantInfo().getPrivilege());
sawRole=true;
}
}
Assert.assertTrue(sawUser && sawRole);
}
APIUtilityVerifier EqualityVerifier roles=HBaseReadWrite.getInstance().getUserRoles(user2);
Assert.assertEquals(2,roles.size());
String[] roleNames=roles.toArray(new String[roles.size()]);
Arrays.sort(roleNames);
Assert.assertArrayEquals(new String[]{roleName1,roleName2},roleNames);
store.removeRole(roleName2);
HBaseReadWrite.setConf(conf);
roles=HBaseReadWrite.getInstance().getUserRoles(user1);
Assert.assertEquals(1,roles.size());
Assert.assertEquals(roleName1,roles.get(0));
roles=HBaseReadWrite.getInstance().getUserRoles(user2);
Assert.assertEquals(1,roles.size());
Assert.assertEquals(roleName1,roles.get(0));
}
APIUtilityVerifier EqualityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableName,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
startTime+=10;
table.setLastAccessTime(startTime);
LOG.debug("XXX alter table test");
store.alterTable("default",tableName,table);
Table t=store.getTable("default",tableName);
LOG.debug("Alter table time " + t.getLastAccessTime());
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals(tableName,t.getTableName());
Assert.assertEquals(startTime,t.getLastAccessTime());
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier dbs=store.getDatabases("db1|db2");
Assert.assertEquals(2,dbs.size());
String[] namesFromStore=dbs.toArray(new String[2]);
Arrays.sort(namesFromStore);
Assert.assertArrayEquals(Arrays.copyOfRange(dbNames,1,3),namesFromStore);
dbs=store.getDatabases("db*");
Assert.assertEquals(3,dbs.size());
namesFromStore=dbs.toArray(new String[3]);
Arrays.sort(namesFromStore);
Assert.assertArrayEquals(dbNames,namesFromStore);
}
InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("mytable","default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
Table t=store.getTable("default","mytable");
Assert.assertEquals(1,t.getSd().getColsSize());
Assert.assertEquals("col1",t.getSd().getCols().get(0).getName());
Assert.assertEquals("int",t.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",t.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",t.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",t.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp",t.getSd().getLocation());
Assert.assertEquals("input",t.getSd().getInputFormat());
Assert.assertEquals("output",t.getSd().getOutputFormat());
Assert.assertEquals("me",t.getOwner());
Assert.assertEquals("default",t.getDbName());
Assert.assertEquals("mytable",t.getTableName());
}
APIUtilityVerifier EqualityVerifier funcs=store.getFunctions(dbname,"func1|func2");
Assert.assertEquals(2,funcs.size());
String[] namesFromStore=funcs.toArray(new String[2]);
Arrays.sort(namesFromStore);
Assert.assertArrayEquals(Arrays.copyOfRange(funcNames,1,3),namesFromStore);
funcs=store.getFunctions(dbname,"func*");
Assert.assertEquals(3,funcs.size());
namesFromStore=funcs.toArray(new String[3]);
Arrays.sort(namesFromStore);
Assert.assertArrayEquals(funcNames,namesFromStore);
funcs=store.getFunctions("nosuchdb","func*");
Assert.assertEquals(0,funcs.size());
}
APIUtilityVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableNames[0],dbName,"me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
table=new Table(tableNames[1],dbName,"me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
String[] roleNames=new String[]{"ltg_role1","ltg_role2"};
String[] userNames=new String[]{"gandalf","radagast"};
store.addRole(roleNames[0],"me");
store.addRole(roleNames[1],"me");
int now=(int)(System.currentTimeMillis() / 1000);
Role role1=store.getRole(roleNames[0]);
Role role2=store.getRole(roleNames[1]);
store.grantRole(role1,userNames[0],PrincipalType.USER,"bob",PrincipalType.USER,false);
store.grantRole(role1,roleNames[1],PrincipalType.ROLE,"admin",PrincipalType.ROLE,true);
store.grantRole(role2,userNames[1],PrincipalType.USER,"bob",PrincipalType.USER,false);
List privileges=new ArrayList();
HiveObjectRef hiveObjRef=new HiveObjectRef(HiveObjectType.TABLE,dbName,tableNames[0],null,null);
PrivilegeGrantInfo grantInfo=new PrivilegeGrantInfo("read",now,"me",PrincipalType.USER,false);
HiveObjectPrivilege hop=new HiveObjectPrivilege(hiveObjRef,userNames[0],PrincipalType.USER,grantInfo);
privileges.add(hop);
grantInfo=new PrivilegeGrantInfo("write",now,"me",PrincipalType.USER,true);
hop=new HiveObjectPrivilege(hiveObjRef,roleNames[0],PrincipalType.ROLE,grantInfo);
privileges.add(hop);
PrivilegeBag pBag=new PrivilegeBag(privileges);
store.grantPrivileges(pBag);
List hops=store.listAllTableGrants(roleNames[0],PrincipalType.ROLE,dbName,tableNames[0]);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.ROLE,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("write",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listAllTableGrants(userNames[0],PrincipalType.USER,dbName,tableNames[0]);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.USER,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("read",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listAllTableGrants(roleNames[1],PrincipalType.ROLE,dbName,tableNames[0]);
Assert.assertEquals(0,hops.size());
hops=store.listAllTableGrants(userNames[1],PrincipalType.USER,dbName,tableNames[0]);
Assert.assertEquals(0,hops.size());
hops=store.listAllTableGrants(roleNames[0],PrincipalType.ROLE,dbName,tableNames[1]);
Assert.assertEquals(0,hops.size());
hops=store.listAllTableGrants(userNames[0],PrincipalType.USER,dbName,tableNames[1]);
Assert.assertEquals(0,hops.size());
hops=store.listTableGrantsAll(dbName,tableNames[0]);
Assert.assertEquals(2,hops.size());
boolean sawUser=false, sawRole=false;
for ( HiveObjectPrivilege h : hops) {
if (h.getPrincipalName().equals(userNames[0])) {
Assert.assertEquals(PrincipalType.USER,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,h.getHiveObject().getObjectType());
Assert.assertEquals("read",h.getGrantInfo().getPrivilege());
sawUser=true;
}
else if (h.getPrincipalName().equals(roleNames[0])) {
Assert.assertEquals(PrincipalType.ROLE,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,h.getHiveObject().getObjectType());
Assert.assertEquals("write",h.getGrantInfo().getPrivilege());
sawRole=true;
}
}
Assert.assertTrue(sawUser && sawRole);
hops=store.listPrincipalTableGrantsAll(roleNames[0],PrincipalType.ROLE);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.ROLE,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("write",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalTableGrantsAll(userNames[0],PrincipalType.USER);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.USER,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.TABLE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("read",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalDBGrantsAll(roleNames[1],PrincipalType.ROLE);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalDBGrantsAll(userNames[1],PrincipalType.USER);
Assert.assertEquals(0,hops.size());
}
finally {
store.dropTable(dbName,tableNames[0]);
store.dropTable(dbName,tableNames[1]);
store.dropDatabase(dbName);
}
}
APIUtilityVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
List partitions=new ArrayList();
for ( String val : partVals) {
List vals=new ArrayList();
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
partitions.add(part);
}
store.addPartitions(dbName,tableName,partitions);
List partNames=store.listPartitionNames(dbName,tableName,(short)-1);
Assert.assertEquals(5,partNames.size());
String[] names=partNames.toArray(new String[partNames.size()]);
Arrays.sort(names);
String[] canonicalNames=partVals.toArray(new String[partVals.size()]);
for (int i=0; i < canonicalNames.length; i++) canonicalNames[i]="pc=" + canonicalNames[i];
Assert.assertArrayEquals(canonicalNames,names);
}
APIUtilityVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table(tableNames[j],dbNames[i],"me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
}
}
List fetchedNames=store.getAllTables(dbNames[0]);
Assert.assertEquals(3,fetchedNames.size());
String[] sortedFetchedNames=fetchedNames.toArray(new String[fetchedNames.size()]);
Arrays.sort(sortedFetchedNames);
Assert.assertArrayEquals(tableNames,sortedFetchedNames);
List regexNames=store.getTables(dbNames[0],"*y");
Assert.assertEquals(2,regexNames.size());
String[] sortedRegexNames=regexNames.toArray(new String[regexNames.size()]);
Arrays.sort(sortedRegexNames);
Assert.assertArrayEquals(Arrays.copyOfRange(tableNames,0,2),sortedRegexNames);
List fetchedTables=store.getTableObjectsByName(dbNames[1],Arrays.asList(Arrays.copyOfRange(tableNames,1,3)));
Assert.assertEquals(2,fetchedTables.size());
sortedFetchedNames=new String[fetchedTables.size()];
for (int i=0; i < fetchedTables.size(); i++) {
sortedFetchedNames[i]=fetchedTables.get(i).getTableName();
}
Arrays.sort(sortedFetchedNames);
Assert.assertArrayEquals(Arrays.copyOfRange(tableNames,1,3),sortedFetchedNames);
}
APIUtilityVerifier IterativeVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema(boolcol,"boolean","nocomment"));
cols.add(new FieldSchema(longcol,"long","nocomment"));
cols.add(new FieldSchema(doublecol,"double","nocomment"));
cols.add(new FieldSchema(stringcol,"varchar(32)","nocomment"));
cols.add(new FieldSchema(binarycol,"binary","nocomment"));
cols.add(new FieldSchema(decimalcol,"decimal(5, 3)","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
Table table=new Table(tableName,dbname,"me",(int)now / 1000,(int)now / 1000,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
for ( String partVal : partVals) {
Partition part=new Partition(Arrays.asList(partVal),dbname,tableName,(int)now / 1000,(int)now / 1000,sd,emptyParameters);
store.addPartition(part);
}
for (int i=0; i < partNames.length; i++) {
ColumnStatistics stats=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName(boolcol);
obj.setColType("boolean");
ColumnStatisticsData data=new ColumnStatisticsData();
BooleanColumnStatsData boolData=new BooleanColumnStatsData();
boolData.setNumTrues(trues);
boolData.setNumFalses(falses);
boolData.setNumNulls(booleanNulls);
data.setBooleanStats(boolData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats,Arrays.asList(partVals[i]));
}
List statsList=store.getPartitionColumnStatistics(dbname,tableName,Arrays.asList(partNames),Arrays.asList(boolcol));
Assert.assertEquals(2,statsList.size());
for (int i=0; i < partNames.length; i++) {
Assert.assertEquals(1,statsList.get(i).getStatsObjSize());
}
for (int i=0; i < partNames.length; i++) {
ColumnStatistics stats=new ColumnStatistics();
ColumnStatisticsDesc desc=new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj=new ColumnStatisticsObj();
obj.setColName(stringcol);
obj.setColType("string");
ColumnStatisticsData data=new ColumnStatisticsData();
StringColumnStatsData strData=new StringColumnStatsData();
strData.setMaxColLen(strMaxLen);
strData.setAvgColLen(strAvgLen);
strData.setNumNulls(strNulls);
strData.setNumDVs(strDVs);
data.setStringStats(strData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats,Arrays.asList(partVals[i]));
}
statsList=store.getPartitionColumnStatistics(dbname,tableName,Arrays.asList(partNames),Arrays.asList(boolcol));
Assert.assertEquals(2,statsList.size());
for (int i=0; i < partNames.length; i++) {
Assert.assertEquals(1,statsList.get(i).getStatsObjSize());
}
statsList=store.getPartitionColumnStatistics(dbname,tableName,Arrays.asList(partNames),Arrays.asList(boolcol,stringcol));
Assert.assertEquals(2,statsList.size());
for (int i=0; i < partNames.length; i++) {
Assert.assertEquals(2,statsList.get(i).getStatsObjSize());
Assert.assertEquals(booleanNulls,statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls());
Assert.assertEquals(strDVs,statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs());
}
}
EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
List partitions=new ArrayList();
List> allVals=new ArrayList>();
for ( String val : partVals) {
List vals=new ArrayList();
allVals.add(vals);
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
partitions.add(part);
}
store.addPartitions(dbName,tableName,partitions);
for ( Partition p : partitions) p.setLastAccessTime(startTime + 10);
store.alterPartitions(dbName,tableName,allVals,partitions);
partitions=store.getPartitions(dbName,tableName,-1);
for ( Partition part : partitions) {
Assert.assertEquals(startTime + 10,part.getLastAccessTime());
}
}
APIUtilityVerifier EqualityVerifier roles=HBaseReadWrite.getInstance().getUserRoles(user1);
Assert.assertEquals(2,roles.size());
String[] roleNames=roles.toArray(new String[roles.size()]);
Arrays.sort(roleNames);
Assert.assertArrayEquals(new String[]{roleName1,roleName2},roleNames);
store.grantRole(role2,user1,PrincipalType.USER,"admin",PrincipalType.ROLE,false);
store.grantRole(role1,user2,PrincipalType.USER,"bob",PrincipalType.USER,false);
HBaseReadWrite.setConf(conf);
roles=HBaseReadWrite.getInstance().getUserRoles(user2);
Assert.assertEquals(2,roles.size());
roleNames=roles.toArray(new String[roles.size()]);
Arrays.sort(roleNames);
Assert.assertArrayEquals(new String[]{roleName1,roleName2},roleNames);
store.revokeRole(role1,roleName2,PrincipalType.ROLE,false);
roles=HBaseReadWrite.getInstance().getUserRoles(user1);
Assert.assertEquals(2,roles.size());
roleNames=roles.toArray(new String[roles.size()]);
Arrays.sort(roleNames);
Assert.assertArrayEquals(new String[]{roleName1,roleName2},roleNames);
roles=HBaseReadWrite.getInstance().getUserRoles(user2);
Assert.assertEquals(1,roles.size());
Assert.assertEquals(roleName1,roles.get(0));
}
BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier privileges=new ArrayList();
HiveObjectRef hiveObjRef=new HiveObjectRef(HiveObjectType.DATABASE,dbNames[0],null,null,null);
PrivilegeGrantInfo grantInfo=new PrivilegeGrantInfo("read",now,"me",PrincipalType.USER,false);
HiveObjectPrivilege hop=new HiveObjectPrivilege(hiveObjRef,userNames[0],PrincipalType.USER,grantInfo);
privileges.add(hop);
grantInfo=new PrivilegeGrantInfo("write",now,"me",PrincipalType.USER,true);
hop=new HiveObjectPrivilege(hiveObjRef,roleNames[0],PrincipalType.ROLE,grantInfo);
privileges.add(hop);
PrivilegeBag pBag=new PrivilegeBag(privileges);
store.grantPrivileges(pBag);
List hops=store.listPrincipalDBGrants(roleNames[0],PrincipalType.ROLE,dbNames[0]);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.ROLE,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("write",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalDBGrants(userNames[0],PrincipalType.USER,dbNames[0]);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.USER,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("read",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalDBGrants(roleNames[1],PrincipalType.ROLE,dbNames[0]);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalDBGrants(userNames[1],PrincipalType.USER,dbNames[0]);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalDBGrants(roleNames[0],PrincipalType.ROLE,dbNames[1]);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalDBGrants(userNames[0],PrincipalType.USER,dbNames[1]);
Assert.assertEquals(0,hops.size());
hops=store.listDBGrantsAll(dbNames[0]);
Assert.assertEquals(2,hops.size());
boolean sawUser=false, sawRole=false;
for ( HiveObjectPrivilege h : hops) {
if (h.getPrincipalName().equals(userNames[0])) {
Assert.assertEquals(PrincipalType.USER,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,h.getHiveObject().getObjectType());
Assert.assertEquals("read",h.getGrantInfo().getPrivilege());
sawUser=true;
}
else if (h.getPrincipalName().equals(roleNames[0])) {
Assert.assertEquals(PrincipalType.ROLE,h.getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,h.getHiveObject().getObjectType());
Assert.assertEquals("write",h.getGrantInfo().getPrivilege());
sawRole=true;
}
}
Assert.assertTrue(sawUser && sawRole);
hops=store.listPrincipalDBGrantsAll(roleNames[0],PrincipalType.ROLE);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.ROLE,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("write",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalDBGrantsAll(userNames[0],PrincipalType.USER);
Assert.assertEquals(1,hops.size());
Assert.assertEquals(PrincipalType.USER,hops.get(0).getPrincipalType());
Assert.assertEquals(HiveObjectType.DATABASE,hops.get(0).getHiveObject().getObjectType());
Assert.assertEquals("read",hops.get(0).getGrantInfo().getPrivilege());
hops=store.listPrincipalDBGrantsAll(roleNames[1],PrincipalType.ROLE);
Assert.assertEquals(0,hops.size());
hops=store.listPrincipalDBGrantsAll(userNames[1],PrincipalType.USER);
Assert.assertEquals(0,hops.size());
}
finally {
store.dropDatabase(dbNames[0]);
store.dropDatabase(dbNames[1]);
}
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List vals=new ArrayList();
vals.add("fred");
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=fred");
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(dbName,tableName,vals);
Assert.assertEquals(1,p.getSd().getColsSize());
Assert.assertEquals("col1",p.getSd().getCols().get(0).getName());
Assert.assertEquals("int",p.getSd().getCols().get(0).getType());
Assert.assertEquals("nocomment",p.getSd().getCols().get(0).getComment());
Assert.assertEquals("serde",p.getSd().getSerdeInfo().getName());
Assert.assertEquals("seriallib",p.getSd().getSerdeInfo().getSerializationLib());
Assert.assertEquals("file:/tmp/pc=fred",p.getSd().getLocation());
Assert.assertEquals("input",p.getSd().getInputFormat());
Assert.assertEquals("output",p.getSd().getOutputFormat());
Assert.assertEquals(dbName,p.getDbName());
Assert.assertEquals(tableName,p.getTableName());
Assert.assertEquals(1,p.getValuesSize());
Assert.assertEquals("fred",p.getValues().get(0));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier dbs=store.getAllDatabases();
Assert.assertEquals(3,dbs.size());
String[] namesFromStore=dbs.toArray(new String[3]);
Arrays.sort(namesFromStore);
Assert.assertArrayEquals(dbNames,namesFromStore);
}
APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier roles=store.listRoles("fred",PrincipalType.USER);
Assert.assertEquals(3,roles.size());
boolean sawRole1=false, sawRole2=false, sawPublic=false;
for ( Role role : roles) {
if (role.getRoleName().equals(roleName1)) {
sawRole1=true;
}
else if (role.getRoleName().equals(roleName2)) {
sawRole2=true;
}
else if (role.getRoleName().equals(HiveMetaStore.PUBLIC)) {
sawPublic=true;
}
else {
Assert.fail("Unknown role name " + role.getRoleName());
}
}
Assert.assertTrue(sawRole1 && sawRole2 && sawPublic);
roles=store.listRoles("fred",PrincipalType.ROLE);
Assert.assertEquals(0,roles.size());
roles=store.listRoles(roleName1,PrincipalType.ROLE);
Assert.assertEquals(1,roles.size());
Role role=roles.get(0);
Assert.assertEquals(roleName2,role.getRoleName());
List grants=store.listRoleMembers(roleName1);
Assert.assertEquals(1,grants.size());
Assert.assertEquals("fred",grants.get(0).getPrincipalName());
Assert.assertEquals(PrincipalType.USER,grants.get(0).getPrincipalType());
Assert.assertTrue("Expected grant time of " + now + " got "+ grants.get(0).getGrantTime(),grants.get(0).getGrantTime() >= now);
Assert.assertEquals("bob",grants.get(0).getGrantorName());
Assert.assertEquals(PrincipalType.USER,grants.get(0).getGrantorPrincipalType());
Assert.assertFalse(grants.get(0).isGrantOption());
grants=store.listRoleMembers(roleName2);
Assert.assertEquals(2,grants.size());
boolean sawFred=false;
sawRole1=false;
for ( RolePrincipalGrant m : grants) {
if ("fred".equals(m.getPrincipalName())) sawFred=true;
else if (roleName1.equals(m.getPrincipalName())) sawRole1=true;
else Assert.fail("Unexpected principal " + m.getPrincipalName());
}
Assert.assertTrue(sawFred && sawRole1);
store.revokeRole(role2,roleName1,PrincipalType.ROLE,true);
roles=store.listRoles(roleName1,PrincipalType.ROLE);
Assert.assertEquals(1,roles.size());
Assert.assertEquals(roleName2,roles.get(0).getRoleName());
grants=store.listRoleMembers(roleName1);
Assert.assertFalse(grants.get(0).isGrantOption());
store.removeRole(roleName1);
roles=store.listRoles("fred",PrincipalType.USER);
Assert.assertEquals(2,roles.size());
sawRole2=sawPublic=false;
for ( Role m : roles) {
if (m.getRoleName().equals(roleName2)) sawRole2=true;
else if (m.getRoleName().equals(HiveMetaStore.PUBLIC)) sawPublic=true;
else Assert.fail("Unknown role " + m.getRoleName());
}
Assert.assertTrue(sawRole2 && sawPublic);
roles=store.listRoles(roleName1,PrincipalType.ROLE);
Assert.assertEquals(0,roles.size());
store.revokeRole(role2,"fred",PrincipalType.USER,false);
roles=store.listRoles("fred",PrincipalType.USER);
Assert.assertEquals(1,roles.size());
Assert.assertEquals(HiveMetaStore.PUBLIC,roles.get(0).getRoleName());
}
APIUtilityVerifier EqualityVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
partCols.add(new FieldSchema("region","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
String[][] partVals=new String[][]{{"today","north america"},{"tomorrow","europe"}};
for ( String[] pv : partVals) {
List vals=new ArrayList();
for ( String v : pv) vals.add(v);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + pv[0] + "/region="+ pv[1]);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
}
List names=store.listPartitionNames(dbName,tableName,(short)-1);
Assert.assertEquals(2,names.size());
String[] resultNames=names.toArray(new String[names.size()]);
Arrays.sort(resultNames);
Assert.assertArrayEquals(resultNames,new String[]{"pc=today/region=north america","pc=tomorrow/region=europe"});
List parts=store.getPartitionsByNames(dbName,tableName,names);
Assert.assertArrayEquals(partVals[0],parts.get(0).getValues().toArray(new String[2]));
Assert.assertArrayEquals(partVals[1],parts.get(1).getValues().toArray(new String[2]));
store.dropPartitions(dbName,tableName,names);
List afterDropParts=store.getPartitions(dbName,tableName,-1);
Assert.assertEquals(0,afterDropParts.size());
}
EqualityVerifier Class: org.apache.hadoop.hive.metastore.hbase.TestSharedStorageDescriptor BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier iter=ssd.getColsIterator();
Assert.assertTrue(iter.hasNext());
Assert.assertEquals("fred",iter.next().getName());
Assert.assertFalse(sd.getCols() == ssd.getCols());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier list=ssd.getBucketCols();
list.add(new String("bob"));
Assert.assertFalse(sd.getBucketCols() == ssd.getBucketCols());
Assert.assertEquals(2,ssd.getBucketColsSize());
Assert.assertEquals(1,sd.getBucketColsSize());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=ssd.getCols();
Assert.assertEquals(1,cols.size());
Assert.assertTrue(sd.getCols() == ssd.getCols());
}
Class: org.apache.hadoop.hive.metastore.hbase.TestStorageDescriptorSharing APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
List partCols=new ArrayList();
partCols.add(new FieldSchema("pc","string",""));
Table table=new Table(tableName,dbName,"me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
store.createTable(table);
List partVals=Arrays.asList("alan","bob","carl","doug","ethan");
for ( String val : partVals) {
List vals=new ArrayList();
vals.add(val);
StorageDescriptor psd=new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + val);
Partition part=new Partition(vals,dbName,tableName,startTime,startTime,psd,emptyParameters);
store.addPartition(part);
Partition p=store.getPartition(dbName,tableName,vals);
Assert.assertEquals("file:/tmp/pc=" + val,p.getSd().getLocation());
}
Assert.assertEquals(1,HBaseReadWrite.getInstance().countStorageDescriptor());
String tableName2="differentTable";
sd=new StorageDescriptor(cols,"file:/tmp","input2","output",false,0,serde,null,null,emptyParameters);
table=new Table(tableName2,"default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
store.createTable(table);
Assert.assertEquals(2,HBaseReadWrite.getInstance().countStorageDescriptor());
store.dropPartition(dbName,tableName,Arrays.asList(partVals.get(0)));
Assert.assertEquals(2,HBaseReadWrite.getInstance().countStorageDescriptor());
table=store.getTable(dbName,tableName2);
byte[] sdHash=HBaseUtils.hashStorageDescriptor(table.getSd(),md);
table.setLastAccessTime(startTime + 1);
store.alterTable(dbName,tableName2,table);
Assert.assertEquals(2,HBaseReadWrite.getInstance().countStorageDescriptor());
table=store.getTable(dbName,tableName2);
byte[] alteredHash=HBaseUtils.hashStorageDescriptor(table.getSd(),md);
Assert.assertArrayEquals(sdHash,alteredHash);
table.getSd().setOutputFormat("output_changed");
store.alterTable(dbName,tableName2,table);
Assert.assertEquals(2,HBaseReadWrite.getInstance().countStorageDescriptor());
table=store.getTable(dbName,tableName2);
alteredHash=HBaseUtils.hashStorageDescriptor(table.getSd(),md);
Assert.assertFalse(Arrays.equals(sdHash,alteredHash));
Partition part=store.getPartition(dbName,tableName,Arrays.asList(partVals.get(1)));
sdHash=HBaseUtils.hashStorageDescriptor(part.getSd(),md);
part.setLastAccessTime(part.getLastAccessTime() + 1);
store.alterPartition(dbName,tableName,Arrays.asList(partVals.get(1)),part);
Assert.assertEquals(2,HBaseReadWrite.getInstance().countStorageDescriptor());
part=store.getPartition(dbName,tableName,Arrays.asList(partVals.get(1)));
alteredHash=HBaseUtils.hashStorageDescriptor(part.getSd(),md);
Assert.assertArrayEquals(sdHash,alteredHash);
part.getSd().setOutputFormat("output_changed_some_more");
store.alterPartition(dbName,tableName,Arrays.asList(partVals.get(1)),part);
Assert.assertEquals(3,HBaseReadWrite.getInstance().countStorageDescriptor());
part=store.getPartition(dbName,tableName,Arrays.asList(partVals.get(1)));
alteredHash=HBaseUtils.hashStorageDescriptor(part.getSd(),md);
Assert.assertFalse(Arrays.equals(sdHash,alteredHash));
List parts=store.getPartitions(dbName,tableName,-1);
sdHash=HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(),md);
for (int i=1; i < 3; i++) {
parts.get(i).setLastAccessTime(97);
}
List> listPartVals=new ArrayList>();
for ( String pv : partVals.subList(1,partVals.size())) {
listPartVals.add(Arrays.asList(pv));
}
store.alterPartitions(dbName,tableName,listPartVals,parts);
Assert.assertEquals(3,HBaseReadWrite.getInstance().countStorageDescriptor());
parts=store.getPartitions(dbName,tableName,-1);
alteredHash=HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(),md);
Assert.assertArrayEquals(sdHash,alteredHash);
parts=store.getPartitions(dbName,tableName,-1);
sdHash=HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(),md);
for (int i=1; i < 3; i++) {
parts.get(i).getSd().setOutputFormat("yet_a_different_of");
}
store.alterPartitions(dbName,tableName,listPartVals,parts);
Assert.assertEquals(4,HBaseReadWrite.getInstance().countStorageDescriptor());
parts=store.getPartitions(dbName,tableName,-1);
alteredHash=HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(),md);
Assert.assertFalse(Arrays.equals(sdHash,alteredHash));
for ( String partVal : partVals.subList(1,partVals.size())) {
store.dropPartition(dbName,tableName,Arrays.asList(partVal));
}
store.dropTable(dbName,tableName);
store.dropTable(dbName,tableName2);
Assert.assertEquals(0,HBaseReadWrite.getInstance().countStorageDescriptor());
}
Class: org.apache.hadoop.hive.metastore.txn.TestCompactionTxnHandler APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(1,compacts.size());
ShowCompactResponseElement c=compacts.get(0);
assertEquals("foo",c.getDbname());
assertEquals("bar",c.getTablename());
assertEquals("ds=today",c.getPartitionname());
assertEquals(CompactionType.MINOR,c.getType());
assertEquals("ready for cleaning",c.getState());
assertNull(c.getWorkerid());
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
txnid=openTxn();
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.DB,"mydb");
comp.setTablename("yourtable");
components=new ArrayList(1);
components.add(comp);
req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
txnid=openTxn();
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.DB,"mydb");
comp.setTablename("foo");
comp.setPartitionname("bar");
components=new ArrayList(1);
components.add(comp);
req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.DB,"mydb");
comp.setTablename("foo");
comp.setPartitionname("baz");
components=new ArrayList(1);
components.add(comp);
req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
CompactionInfo ci=new CompactionInfo();
CompactionRequest rqst=new CompactionRequest("mydb","mytable",CompactionType.MAJOR);
txnHandler.compact(rqst);
assertEquals(0,txnHandler.findReadyToClean().size());
ci=txnHandler.findNextToCompact("fred");
assertNotNull(ci);
txnHandler.markCompacted(ci);
List toClean=txnHandler.findReadyToClean();
assertEquals(1,toClean.size());
txnHandler.markCleaned(ci);
GetOpenTxnsResponse txnList=txnHandler.getOpenTxns();
assertEquals(3,txnList.getOpen_txnsSize());
txnHandler.cleanEmptyAbortedTxns();
txnList=txnHandler.getOpenTxns();
assertEquals(2,txnList.getOpen_txnsSize());
rqst=new CompactionRequest("mydb","foo",CompactionType.MAJOR);
rqst.setPartitionname("bar");
txnHandler.compact(rqst);
assertEquals(0,txnHandler.findReadyToClean().size());
ci=txnHandler.findNextToCompact("fred");
assertNotNull(ci);
txnHandler.markCompacted(ci);
toClean=txnHandler.findReadyToClean();
assertEquals(1,toClean.size());
txnHandler.markCleaned(ci);
txnHandler.openTxns(new OpenTxnRequest(1,"me","localhost"));
txnHandler.cleanEmptyAbortedTxns();
txnList=txnHandler.getOpenTxns();
assertEquals(3,txnList.getOpen_txnsSize());
}
BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(3,compacts.size());
boolean sawWorkingBob=false;
int initiatedCount=0;
for ( ShowCompactResponseElement c : compacts) {
if (c.getState().equals("working")) {
assertEquals("bob-193892",c.getWorkerid());
sawWorkingBob=true;
}
else if (c.getState().equals("initiated")) {
initiatedCount++;
}
else {
fail("Unexpected state");
}
}
assertTrue(sawWorkingBob);
assertEquals(2,initiatedCount);
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier toClean=txnHandler.findReadyToClean();
assertEquals(1,toClean.size());
assertNull(txnHandler.findNextToCompact("fred"));
txnHandler.markCleaned(ci);
assertNull(txnHandler.findNextToCompact("fred"));
assertEquals(0,txnHandler.findReadyToClean().size());
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
assertEquals(1,rsp.getCompactsSize());
assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier toClean=txnHandler.findReadyToClean();
assertEquals(1,toClean.size());
assertNull(txnHandler.findNextToCompact("fred"));
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
assertEquals(1,compacts.size());
ShowCompactResponseElement c=compacts.get(0);
assertEquals("foo",c.getDbname());
assertEquals("bar",c.getTablename());
assertEquals("ds=today",c.getPartitionname());
assertEquals(CompactionType.MINOR,c.getType());
assertEquals("ready for cleaning",c.getState());
assertNull(c.getWorkerid());
}
BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.DB,"mydb");
comp.setTablename("yourtable");
comp.setPartitionname("mypartition");
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
assertEquals(0,txnHandler.numLocksInLockTable());
Set potentials=txnHandler.findPotentialCompactions(100);
assertEquals(2,potentials.size());
boolean sawMyTable=false, sawYourTable=false;
for ( CompactionInfo ci : potentials) {
sawMyTable|=(ci.dbname.equals("mydb") && ci.tableName.equals("mytable") && ci.partName == null);
sawYourTable|=(ci.dbname.equals("mydb") && ci.tableName.equals("yourtable") && ci.partName.equals("mypartition"));
}
assertTrue(sawMyTable);
assertTrue(sawYourTable);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(1,compacts.size());
ShowCompactResponseElement c=compacts.get(0);
assertEquals("foo",c.getDbname());
assertEquals("bar",c.getTablename());
assertEquals("ds=today",c.getPartitionname());
assertEquals(CompactionType.MINOR,c.getType());
assertEquals("working",c.getState());
assertTrue(c.getStart() - 5000 < now && c.getStart() + 5000 > now);
assertEquals("fred",c.getWorkerid());
assertEquals("bob",c.getRunAs());
}
BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(2,compacts.size());
boolean sawWorking=false, sawInitiated=false;
for ( ShowCompactResponseElement c : compacts) {
if (c.getState().equals("working")) sawWorking=true;
else if (c.getState().equals("initiated")) sawInitiated=true;
else fail("Unexpected state");
}
assertTrue(sawWorking);
assertTrue(sawInitiated);
}
BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(2,compacts.size());
for ( ShowCompactResponseElement e : compacts) {
assertEquals("working",e.getState());
assertTrue(e.getStart() - 5000 < now && e.getStart() + 5000 > now);
assertEquals("fred",e.getWorkerid());
}
}
APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier potentials=txnHandler.findPotentialCompactions(1000);
assertEquals(2,potentials.size());
SortedSet sorted=new TreeSet(potentials);
int i=0;
for ( CompactionInfo ci : sorted) {
assertEquals(dbName,ci.dbname);
assertEquals(tableName,ci.tableName);
switch (i++) {
case 0:
assertEquals("ds=today",ci.partName);
break;
case 1:
assertEquals("ds=yesterday",ci.partName);
break;
default :
throw new RuntimeException("What?");
}
}
}
Class: org.apache.hadoop.hive.metastore.txn.TestTxnHandler InternalCallVerifier EqualityVerifier IterativeVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnid=openTxn();
comp=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"mydb");
comp.setTablename("mytable");
components=new ArrayList(1);
components.add(comp);
req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
res=txnHandler.lock(req);
components=new ArrayList(1);
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.PARTITION,"yourdb");
comp.setTablename("yourtable");
comp.setPartitionname("yourpartition");
components.add(comp);
req=new LockRequest(components,"you","remotehost");
res=txnHandler.lock(req);
ShowLocksResponse rsp=txnHandler.showLocks(new ShowLocksRequest());
List locks=rsp.getLocks();
assertEquals(3,locks.size());
boolean[] saw=new boolean[locks.size()];
for (int i=0; i < saw.length; i++) saw[i]=false;
for ( ShowLocksResponseElement lock : locks) {
if (lock.getLockid() == 1) {
assertEquals(1,lock.getTxnid());
assertEquals("mydb",lock.getDbname());
assertNull(lock.getTablename());
assertNull(lock.getPartname());
assertEquals(LockState.ACQUIRED,lock.getState());
assertEquals(LockType.EXCLUSIVE,lock.getType());
assertTrue(lock.toString(),0 == lock.getLastheartbeat() && lock.getTxnid() != 0);
assertTrue("Expected acquired at " + lock.getAcquiredat() + " to be between "+ begining+ " and "+ System.currentTimeMillis(),begining <= lock.getAcquiredat() && System.currentTimeMillis() >= lock.getAcquiredat());
assertEquals("me",lock.getUser());
assertEquals("localhost",lock.getHostname());
saw[0]=true;
}
else if (lock.getLockid() == 2) {
assertEquals(2,lock.getTxnid());
assertEquals("mydb",lock.getDbname());
assertEquals("mytable",lock.getTablename());
assertNull(lock.getPartname());
assertEquals(LockState.WAITING,lock.getState());
assertEquals(LockType.SHARED_READ,lock.getType());
assertTrue(lock.toString(),0 == lock.getLastheartbeat() && lock.getTxnid() != 0);
assertEquals(0,lock.getAcquiredat());
assertEquals("me",lock.getUser());
assertEquals("localhost",lock.getHostname());
saw[1]=true;
}
else if (lock.getLockid() == 3) {
assertEquals(0,lock.getTxnid());
assertEquals("yourdb",lock.getDbname());
assertEquals("yourtable",lock.getTablename());
assertEquals("yourpartition",lock.getPartname());
assertEquals(LockState.ACQUIRED,lock.getState());
assertEquals(LockType.SHARED_WRITE,lock.getType());
assertTrue(lock.toString(),begining <= lock.getLastheartbeat() && System.currentTimeMillis() >= lock.getLastheartbeat());
assertTrue(begining <= lock.getAcquiredat() && System.currentTimeMillis() >= lock.getAcquiredat());
assertEquals("you",lock.getUser());
assertEquals("remotehost",lock.getHostname());
saw[2]=true;
}
else {
fail("Unknown lock id");
}
}
for (int i=0; i < saw.length; i++) assertTrue("Didn't see lock id " + i,saw[i]);
}
EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
assertEquals(0,txnHandler.numLocksInLockTable());
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier txnList=openedTxns.getTxn_ids();
long first=txnList.get(0);
assertEquals(1L,first);
long second=txnList.get(1);
assertEquals(2L,second);
txnHandler.abortTxn(new AbortTxnRequest(1));
GetOpenTxnsInfoResponse txnsInfo=txnHandler.getOpenTxnsInfo();
assertEquals(2L,txnsInfo.getTxn_high_water_mark());
assertEquals(2,txnsInfo.getOpen_txns().size());
assertEquals(1L,txnsInfo.getOpen_txns().get(0).getId());
assertEquals(TxnState.ABORTED,txnsInfo.getOpen_txns().get(0).getState());
assertEquals(2L,txnsInfo.getOpen_txns().get(1).getId());
assertEquals(TxnState.OPEN,txnsInfo.getOpen_txns().get(1).getState());
GetOpenTxnsResponse txns=txnHandler.getOpenTxns();
assertEquals(2L,txns.getTxn_high_water_mark());
assertEquals(2,txns.getOpen_txns().size());
boolean[] saw=new boolean[3];
for (int i=0; i < saw.length; i++) saw[i]=false;
for ( Long tid : txns.getOpen_txns()) {
saw[tid.intValue()]=true;
}
for (int i=1; i < saw.length; i++) assertTrue(saw[i]);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier compacts=rsp.getCompacts();
assertEquals(1,compacts.size());
ShowCompactResponseElement c=compacts.get(0);
assertEquals("foo",c.getDbname());
assertEquals("bar",c.getTablename());
assertNull(c.getPartitionname());
assertEquals(CompactionType.MINOR,c.getType());
assertEquals("initiated",c.getState());
assertEquals(0L,c.getStart());
assertEquals("fred",c.getRunAs());
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
assertEquals(0,txnHandler.numLocksInLockTable());
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=new ArrayList(2);
components.add(comp);
comp=new LockComponent(LockType.EXCLUSIVE,LockLevel.DB,"mydb");
comp.setTablename("mytable");
comp.setPartitionname("anotherpartition");
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
LockResponse res=txnHandler.lock(req);
long lockid=res.getLockid();
assertTrue(res.getState() == LockState.ACQUIRED);
res=txnHandler.checkLock(new CheckLockRequest(lockid));
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.unlock(new UnlockRequest(lockid));
assertEquals(0,txnHandler.numLocksInLockTable());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier compacts=rsp.getCompacts();
assertEquals(1,compacts.size());
ShowCompactResponseElement c=compacts.get(0);
assertEquals("foo",c.getDbname());
assertEquals("bar",c.getTablename());
assertEquals("ds=today",c.getPartitionname());
assertEquals(CompactionType.MAJOR,c.getType());
assertEquals("initiated",c.getState());
assertEquals(0L,c.getStart());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.metastore.txn.TestValidCompactorTxnList InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.TestCreateUdfEntities InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.TestTxnCommands EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData={{1,7},{3,7},{5,2},{5,3}};
Assert.assertEquals("Bulk update failed",stringifyValues(updatedData),rs);
}
APIUtilityVerifier EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match rs0",stringifyValues(rows1),rs0);
runStatementOnDriver("set autocommit false");
runStatementOnDriver("START TRANSACTION");
int[][] rows2={{5,6},{7,8}};
runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) "+ makeValuesClause(rows2));
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
List allData=stringifyValues(rows1);
allData.addAll(stringifyValues(rows2));
Assert.assertEquals("Content didn't match rs1",allData,rs1);
runStatementOnDriver("update " + Table.ACIDTBL + " set b = 1 where b != 1");
int[][] updatedData={{1,1},{3,1},{5,1},{7,1}};
List rs2=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after update",stringifyValues(updatedData),rs2);
runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 7 and b = 1");
dumpTableData(Table.ACIDTBL,1,0);
dumpTableData(Table.ACIDTBL,2,0);
dumpTableData(Table.ACIDTBL,2,2);
dumpTableData(Table.ACIDTBL,2,4);
int[][] updatedData2={{1,1},{3,1},{5,1}};
List rs3=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after delete",stringifyValues(updatedData2),rs3);
runStatementOnDriver("commit");
runStatementOnDriver("set autocommit true");
List rs4=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after commit",stringifyValues(updatedData2),rs4);
}
EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Can't see my own write",1,rs0.size());
CommandProcessorResponse cpr=runStatementOnDriverNegative("select * from no_such_table");
Assert.assertEquals("Txn didn't fail?","FAILED: SemanticException [Error 10001]: Line 1:14 Table not found 'no_such_table'",cpr.getErrorMessage());
runStatementOnDriver("start transaction");
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
runStatementOnDriver("commit");
Assert.assertEquals("Didn't rollback as expected",0,rs1.size());
}
EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match rs0",stringifyValues(rows1),rs0);
runStatementOnDriver("set autocommit false");
runStatementOnDriver("START TRANSACTION");
runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 8");
int[][] updatedData2={{1,2},{3,4},{5,6}};
List rs2=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after delete",stringifyValues(updatedData2),rs2);
runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4");
int[][] updatedData3={{1,2},{5,6}};
List rs3=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after delete2",stringifyValues(updatedData3),rs3);
runStatementOnDriver("update " + Table.ACIDTBL + " set b=3");
dumpTableData(Table.ACIDTBL,1,0);
dumpTableData(Table.ACIDTBL,2,0);
dumpTableData(Table.ACIDTBL,2,2);
dumpTableData(Table.ACIDTBL,2,4);
List rs5=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData4={{1,3},{5,3}};
Assert.assertEquals("Wrong data after delete",stringifyValues(updatedData4),rs5);
runStatementOnDriver("commit");
runStatementOnDriver("set autocommit true");
List rs4=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after commit",stringifyValues(updatedData4),rs4);
}
EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Rollback didn't rollback",0,rs.size());
}
APIUtilityVerifier EqualityVerifier allData=stringifyValues(rows1);
allData.addAll(stringifyValues(rows2));
List rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match before commit rs",allData,rs);
runStatementOnDriver("commit");
dumpTableData(Table.ACIDTBL,1,0);
dumpTableData(Table.ACIDTBL,1,1);
runStatementOnDriver("set autocommit true");
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match after commit rs1",allData,rs1);
}
APIUtilityVerifier EqualityVerifier allData=stringifyValues(rows1);
allData.addAll(stringifyValues(rows2));
List rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Data didn't match inside tx (rs0)",allData,rs0);
runStatementOnDriver("COMMIT WORK");
dumpTableData(Table.ACIDTBL,1,0);
dumpTableData(Table.ACIDTBL,2,0);
runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
runStatementOnDriver("COMMIT");
runStatementOnDriver("set autocommit true");
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Data didn't match inside tx (rs0)",allData,rs1);
}
EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match rs0",stringifyValues(rows1),rs0);
runStatementOnDriver("set autocommit false");
runStatementOnDriver("START TRANSACTION");
runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4");
int[][] updatedData2={{1,2}};
List rs3=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after delete",stringifyValues(updatedData2),rs3);
runStatementOnDriver("commit");
runStatementOnDriver("set autocommit true");
List rs4=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after commit",stringifyValues(updatedData2),rs4);
}
EqualityVerifier rs=runStatementOnDriver("select * from " + Table.ACIDTBL);
Assert.assertEquals("Expected empty " + Table.ACIDTBL,0,rs.size());
runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
List rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Can't see my own write",1,rs0.size());
runStatementOnDriver("commit");
runStatementOnDriver("START TRANSACTION");
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
runStatementOnDriver("rollback work");
Assert.assertEquals("Can't see write after commit",1,rs1.size());
}
APIUtilityVerifier EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Content didn't match rs0",stringifyValues(rows1),rs0);
runStatementOnDriver("set autocommit false");
runStatementOnDriver("START TRANSACTION");
int[][] rows2={{5,6},{7,8}};
runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) "+ makeValuesClause(rows2));
List rs1=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
List allData=stringifyValues(rows1);
allData.addAll(stringifyValues(rows2));
Assert.assertEquals("Content didn't match rs1",allData,rs1);
runStatementOnDriver("update " + Table.ACIDTBL + " set b = 1 where b != 1");
int[][] updatedData={{1,1},{3,1},{5,1},{7,1}};
List rs2=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after update",stringifyValues(updatedData),rs2);
runStatementOnDriver("commit");
runStatementOnDriver("set autocommit true");
List rs4=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Wrong data after commit",stringifyValues(updatedData),rs4);
}
InternalCallVerifier EqualityVerifier rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Can't see my own write",1,rs0.size());
runStatementOnDriver("set autocommit true");
rs0=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
Assert.assertEquals("Can't see my own write",1,rs0.size());
}
Class: org.apache.hadoop.hive.ql.TestTxnCommands2 EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData={{1,7},{3,3},{5,3}};
Assert.assertEquals("Update failed",stringifyValues(updatedData),rs);
runStatementOnDriver("update " + Table.ACIDTBL + " set B = B + 1 where A=1");
List rs2=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData2={{1,8},{3,3},{5,3}};
Assert.assertEquals("Update failed",stringifyValues(updatedData2),rs2);
}
EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL);
Assert.assertEquals("Insert into " + Table.ACIDTBL + " didn't match:",stringifyValues(tableData),rs);
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) select a,b from "+ Table.ACIDTBLPART+ " where p = 1");
List rs2=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
Assert.assertEquals("Insert into " + Table.NONACIDORCTBL + " didn't match:",stringifyValues(tableData),rs2);
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
int[][] resultData=new int[][]{{1,2}};
Assert.assertEquals(stringifyValues(resultData),rs);
rs=runStatementOnDriver("select count(*) from " + Table.NONACIDORCTBL);
int resultCount=1;
Assert.assertEquals(resultCount,Integer.parseInt(rs.get(0)));
runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true')");
status=fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT,status.length);
for (int i=0; i < status.length; i++) {
Assert.assertTrue(status[i].getPath().getName().matches("00000[01]_0"));
}
rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData=new int[][]{{1,2}};
Assert.assertEquals(stringifyValues(resultData),rs);
rs=runStatementOnDriver("select count(*) from " + Table.NONACIDORCTBL);
resultCount=1;
Assert.assertEquals(resultCount,Integer.parseInt(rs.get(0)));
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(3,4)");
status=fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(3,status.length);
boolean sawNewDelta=false;
for (int i=0; i < status.length; i++) {
if (status[i].getPath().getName().matches("delta_.*")) {
sawNewDelta=true;
FileStatus[] buckets=fs.listStatus(status[i].getPath(),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
}
else {
Assert.assertTrue(status[i].getPath().getName().matches("00000[01]_0"));
}
}
Assert.assertTrue(sawNewDelta);
rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData=new int[][]{{1,2},{3,4}};
Assert.assertEquals(stringifyValues(resultData),rs);
rs=runStatementOnDriver("select count(*) from " + Table.NONACIDORCTBL);
resultCount=2;
Assert.assertEquals(resultCount,Integer.parseInt(rs.get(0)));
runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " compact 'MAJOR'");
Worker w=new Worker();
w.setThreadId((int)w.getId());
w.setHiveConf(hiveConf);
AtomicBoolean stop=new AtomicBoolean();
AtomicBoolean looped=new AtomicBoolean();
stop.set(true);
w.init(stop,looped);
w.run();
status=fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(4,status.length);
boolean sawNewBase=false;
for (int i=0; i < status.length; i++) {
if (status[i].getPath().getName().matches("base_.*")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(status[i].getPath(),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
}
}
Assert.assertTrue(sawNewBase);
rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData=new int[][]{{1,2},{3,4}};
Assert.assertEquals(stringifyValues(resultData),rs);
rs=runStatementOnDriver("select count(*) from " + Table.NONACIDORCTBL);
resultCount=2;
Assert.assertEquals(resultCount,Integer.parseInt(rs.get(0)));
String fakeFile0=TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()+ "/subdir/000000_0";
String fakeFile1=TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()+ "/subdir/000000_1";
fs.create(new Path(fakeFile0));
fs.create(new Path(fakeFile1));
status=fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(5,status.length);
Cleaner c=new Cleaner();
c.setThreadId((int)c.getId());
c.setHiveConf(hiveConf);
stop=new AtomicBoolean();
looped=new AtomicBoolean();
stop.set(true);
c.init(stop,looped);
c.run();
status=fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(1,status.length);
Assert.assertTrue(status[0].getPath().getName().matches("base_.*"));
FileStatus[] buckets=fs.listStatus(status[0].getPath(),FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData=new int[][]{{1,2},{3,4}};
Assert.assertEquals(stringifyValues(resultData),rs);
rs=runStatementOnDriver("select count(*) from " + Table.NONACIDORCTBL);
resultCount=2;
Assert.assertEquals(resultCount,Integer.parseInt(rs.get(0)));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier expected=new ArrayList<>();
expected.add("1\tfoo");
expected.add("2\tbar");
expected.add("3\tblah");
Assert.assertEquals("",expected,runStatementOnDriver("select a,b from " + tblName + " order by a"));
Cleaner c=new Cleaner();
c.setThreadId((int)c.getId());
c.setHiveConf(hiveConf);
c.init(stop,new AtomicBoolean());
c.run();
Initiator i=new Initiator();
i.setThreadId((int)i.getId());
i.setHiveConf(hiveConf);
i.init(stop,new AtomicBoolean());
i.run();
Assert.assertEquals("",expected,runStatementOnDriver("select a,b from " + tblName + " order by a"));
}
EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL + " order by a,b");
int[][] joinData={{2,9}};
Assert.assertEquals("Self join non-part insert overwrite failed",stringifyValues(joinData),rs);
int[][] part2Data={{1,8}};
runStatementOnDriver("insert into " + Table.NONACIDPART + " partition(p=1) (a,b) "+ makeValuesClause(part1Data));
runStatementOnDriver("insert into " + Table.NONACIDPART + " partition(p=2) (a,b) "+ makeValuesClause(part2Data));
runStatementOnDriver("insert overwrite table " + Table.NONACIDPART + " partition(p=1) select a,b from "+ Table.NONACIDPART);
List rs2=runStatementOnDriver("select a,b from " + Table.NONACIDPART + " order by a,b");
int[][] updatedData={{1,7},{1,8},{1,8}};
Assert.assertEquals("Insert overwrite partition failed",stringifyValues(updatedData),rs2);
}
EqualityVerifier rs=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData={{1,7},{3,7},{5,2},{5,3}};
Assert.assertEquals("Bulk update failed",stringifyValues(updatedData),rs);
runStatementOnDriver("update " + Table.ACIDTBL + " set b=19 where b in(select b from "+ Table.NONACIDORCTBL+ " where a = 3)");
List rs2=runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
int[][] updatedData2={{1,19},{3,19},{5,2},{5,3}};
Assert.assertEquals("Bulk update2 failed",stringifyValues(updatedData2),rs2);
}
Class: org.apache.hadoop.hive.ql.exec.TestFileSinkOperator EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.TestOperators InternalCallVerifier EqualityVerifier list=new ArrayList();
list.add(new Path("hdfs://nn.example.com/fi\tl\\e\t1"));
list.add(new Path("hdfs://nn.example.com/file\t2"));
list.add(new Path("file:/file3"));
FetchOperator.setFetchOperatorContext(conf,list);
String[] parts=conf.get(FetchOperator.FETCH_OPERATOR_DIRECTORY_LIST).split("\t");
assertEquals(3,parts.length);
assertEquals("hdfs://nn.example.com/fi\\tl\\\\e\\t1",parts[0]);
assertEquals("hdfs://nn.example.com/file\\t2",parts[1]);
assertEquals("file:/file3",parts[2]);
}
InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.errors.TestTaskLogProcessor InternalCallVerifier EqualityVerifier > stackTraces=taskLogProcessor.getStackTraces();
assertEquals(3,stackTraces.size());
checkException(oomeStr,stackTraces.get(0));
checkException(compositeStr,stackTraces.get(1));
checkException(eofeStr,stackTraces.get(2));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier errList=taskLogProcessor.getErrors();
assertEquals(1,errList.size());
final ErrorAndSolution eas=errList.get(0);
String error=eas.getError();
assertNotNull(error);
assertTrue(error.contains(badFile1) || error.contains(badFile2));
String solution=eas.getSolution();
assertNotNull(solution);
assertTrue(solution.length() > 0);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier errList=taskLogProcessor.getErrors();
assertEquals(1,errList.size());
final ErrorAndSolution eas=errList.get(0);
String error=eas.getError();
assertNotNull(error);
assertTrue(error.contains("memory"));
String solution=eas.getSolution();
assertNotNull(solution);
assertTrue(solution.length() > 0);
String confName=HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY.toString();
assertTrue(solution.contains(confName));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier errList=taskLogProcessor.getErrors();
assertEquals(1,errList.size());
final ErrorAndSolution eas=errList.get(0);
String error=eas.getError();
assertNotNull(error);
assertTrue(error.indexOf(errorCode) >= 0);
String solution=eas.getSolution();
assertNotNull(solution);
assertTrue(solution.length() > 0);
}
Class: org.apache.hadoop.hive.ql.exec.persistence.TestBytesBytesMultiHashMap InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.persistence.TestMapJoinEqualityTableContainer BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier localContainer=new HashMap();
for ( Entry entry : container.entrySet()) {
localContainer.put(entry.getKey(),entry.getValue());
}
Utilities.testEquality(container.get(KEY1),localContainer.get(KEY1));
Utilities.testEquality(container.get(KEY2),localContainer.get(KEY2));
Utilities.testEquality(container.get(KEY3),localContainer.get(KEY3));
Utilities.testEquality(container.get(KEY4),localContainer.get(KEY4));
container.clear();
Assert.assertEquals(0,container.size());
Assert.assertTrue(container.entrySet().isEmpty());
}
Class: org.apache.hadoop.hive.ql.exec.persistence.TestMapJoinRowContainer APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.persistence.TestMapJoinTableContainer APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.tez.InputSplitComparatorTest EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.tez.TestHostAffinitySplitLocationProvider InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.tez.TestTezSessionPool IterativeVerifier UtilityVerifier EqualityVerifier HybridVerifier BranchVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.tez.TestTezTask InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier resources=Collections.singletonList(res);
final Map resMap=new HashMap();
resMap.put("foo.jar",res);
when(utils.localizeTempFiles(path.toString(),conf,inputOutputJars)).thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
assertEquals(resMap,task.getExtraLocalResources(conf,path,inputOutputJars));
}
Class: org.apache.hadoop.hive.ql.exec.vector.TestListColumnVector InternalCallVerifier EqualityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.TestMapColumnVector InternalCallVerifier EqualityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.TestStructColumnVector InternalCallVerifier EqualityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.TestUnionColumnVector InternalCallVerifier EqualityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.TestVectorFilterOperator InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.vector.TestVectorizationContext InternalCallVerifier EqualityVerifier children=new ArrayList(2);
ExprNodeConstantDesc constDesc=new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo,20);
ExprNodeColumnDesc colDesc=new ExprNodeColumnDesc(Boolean.class,"a","table",false);
children.add(colDesc);
children.add(constDesc);
colEqualScalar.setChildren(children);
List columns=new ArrayList();
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(colEqualScalar,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterLongColEqualLongScalar.class,ve.getClass());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier children1=new ArrayList();
List children2=new ArrayList();
children1.add(colDesc1);
children2.add(colDesc2);
List columns=new ArrayList();
columns.add("b");
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
GenericUDFBridge gudfBridge=new GenericUDFBridge("sin",false,UDFSin.class.getName());
mathFuncExpr.setGenericUDF(gudfBridge);
mathFuncExpr.setChildren(children2);
VectorExpression ve=vc.getVectorExpression(mathFuncExpr,VectorExpressionDescriptor.Mode.PROJECTION);
Assert.assertEquals(FuncSinDoubleToDouble.class,ve.getClass());
GenericUDFRound udfRound=new GenericUDFRound();
mathFuncExpr.setGenericUDF(udfRound);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncRoundDoubleToDouble.class,ve.getClass());
GenericUDFBRound udfBRound=new GenericUDFBRound();
mathFuncExpr.setGenericUDF(udfBRound);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncBRoundDoubleToDouble.class,ve.getClass());
mathFuncExpr.setGenericUDF(udfRound);
children2.add(new ExprNodeConstantDesc(4));
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(RoundWithNumDigitsDoubleToDouble.class,ve.getClass());
Assert.assertEquals(4,((RoundWithNumDigitsDoubleToDouble)ve).getDecimalPlaces().get());
mathFuncExpr.setGenericUDF(udfBRound);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(BRoundWithNumDigitsDoubleToDouble.class,ve.getClass());
Assert.assertEquals(4,((BRoundWithNumDigitsDoubleToDouble)ve).getDecimalPlaces().get());
gudfBridge=new GenericUDFBridge("log",false,UDFLog.class.getName());
mathFuncExpr.setGenericUDF(gudfBridge);
children2.clear();
children2.add(new ExprNodeConstantDesc(4.0));
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class,ve.getClass());
Assert.assertTrue(4 == ((FuncLogWithBaseDoubleToDouble)ve).getBase());
children2.clear();
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLnDoubleToDouble.class,ve.getClass());
children2.clear();
children2.add(new ExprNodeConstantDesc(4.5));
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseDoubleToDouble.class,ve.getClass());
Assert.assertTrue(4.5 == ((FuncLogWithBaseDoubleToDouble)ve).getBase());
children2.clear();
children2.add(new ExprNodeConstantDesc(4.5));
children2.add(colDesc1);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncLogWithBaseLongToDouble.class,ve.getClass());
Assert.assertTrue(4.5 == ((FuncLogWithBaseLongToDouble)ve).getBase());
children2.clear();
children2.add(colDesc2);
children2.add(new ExprNodeConstantDesc(4.5));
mathFuncExpr.setGenericUDF(new GenericUDFPower());
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncPowerDoubleToDouble.class,ve.getClass());
Assert.assertTrue(4.5 == ((FuncPowerDoubleToDouble)ve).getPower());
mathFuncExpr.setGenericUDF(udfRound);
children2.clear();
children2.add(colDesc2);
mathFuncExpr.setChildren(children2);
ve=vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncRoundDoubleToDouble.class,ve.getClass());
}
EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier children=new ArrayList();
children.add(colDesc);
stringUnary.setChildren(children);
List columns=new ArrayList();
columns.add("b");
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
GenericUDF stringLower=new GenericUDFLower();
stringUnary.setGenericUDF(stringLower);
VectorExpression ve=vc.getVectorExpression(stringUnary);
assertEquals(StringLower.class,ve.getClass());
assertEquals(1,((StringLower)ve).getColNum());
assertEquals(2,((StringLower)ve).getOutputColumn());
vc=new VectorizationContext("name",columns);
ExprNodeGenericFuncDesc anotherUnary=new ExprNodeGenericFuncDesc();
anotherUnary.setTypeInfo(TypeInfoFactory.stringTypeInfo);
List children2=new ArrayList();
children2.add(stringUnary);
anotherUnary.setChildren(children2);
GenericUDFBridge udfbridge=new GenericUDFBridge("ltrim",false,GenericUDFLTrim.class.getName());
anotherUnary.setGenericUDF(udfbridge);
ve=vc.getVectorExpression(anotherUnary);
VectorExpression childVe=ve.getChildExpressions()[0];
assertEquals(StringLower.class,childVe.getClass());
assertEquals(1,((StringLower)childVe).getColNum());
assertEquals(2,((StringLower)childVe).getOutputColumn());
assertEquals(StringLTrim.class,ve.getClass());
assertEquals(2,((StringLTrim)ve).getInputColumn());
assertEquals(3,((StringLTrim)ve).getOutputColumn());
}
EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier children=new ArrayList();
children.add(colDesc1);
List columns=new ArrayList();
columns.add("b");
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
GenericUDFBridge gudfBridge=new GenericUDFBridge("year",false,UDFYear.class.getName());
tsFuncExpr.setGenericUDF(gudfBridge);
tsFuncExpr.setChildren(children);
VectorExpression ve=vc.getVectorExpression(tsFuncExpr);
Assert.assertEquals(VectorUDFYearLong.class,ve.getClass());
GenericUDFToUnixTimeStamp gudf=new GenericUDFToUnixTimeStamp();
tsFuncExpr.setGenericUDF(gudf);
tsFuncExpr.setTypeInfo(TypeInfoFactory.longTypeInfo);
ve=vc.getVectorExpression(tsFuncExpr);
Assert.assertEquals(VectorUDFUnixTimeStampLong.class,ve.getClass());
}
InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeGenericFuncDesc isNotNullExpr=new ExprNodeGenericFuncDesc();
isNotNullExpr.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
GenericUDFOPNotNull notNullUdf=new GenericUDFOPNotNull();
isNotNullExpr.setGenericUDF(notNullUdf);
List childOfNot=new ArrayList();
childOfNot.add(greaterExprDesc);
isNotNullExpr.setChildren(childOfNot);
List columns=new ArrayList();
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(isNotNullExpr,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(),SelectColumnIsNotNull.class);
assertEquals(2,((SelectColumnIsNotNull)ve).getColNum());
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
ve=vc.getVectorExpression(isNotNullExpr,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(),IsNotNull.class);
assertEquals(2,((IsNotNull)ve).getColNum());
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier children1=new ArrayList();
children1.add(new ExprNodeConstantDesc(new Boolean(false)));
children1.add(col1Expr);
children1.add(constDesc);
children1.add(constDesc2);
ExprNodeGenericFuncDesc exprDesc=new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,udf,children1);
List columns=new ArrayList();
columns.add("col0");
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterStringColumnBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterStringColumnNotBetween);
CharTypeInfo charTypeInfo=new CharTypeInfo(10);
col1Expr=new ExprNodeColumnDesc(charTypeInfo,"col1","table",false);
constDesc=new ExprNodeConstantDesc(charTypeInfo,new HiveChar("Alpha",10));
constDesc2=new ExprNodeConstantDesc(charTypeInfo,new HiveChar("Bravo",10));
udf=new GenericUDFBetween();
children1=new ArrayList();
children1.add(new ExprNodeConstantDesc(new Boolean(false)));
children1.add(col1Expr);
children1.add(constDesc);
children1.add(constDesc2);
exprDesc=new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,udf,children1);
vc=new VectorizationContext("name",columns);
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterCharColumnBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterCharColumnNotBetween);
VarcharTypeInfo varcharTypeInfo=new VarcharTypeInfo(10);
col1Expr=new ExprNodeColumnDesc(varcharTypeInfo,"col1","table",false);
constDesc=new ExprNodeConstantDesc(varcharTypeInfo,new HiveVarchar("Alpha",10));
constDesc2=new ExprNodeConstantDesc(varcharTypeInfo,new HiveVarchar("Bravo",10));
udf=new GenericUDFBetween();
children1=new ArrayList();
children1.add(new ExprNodeConstantDesc(new Boolean(false)));
children1.add(col1Expr);
children1.add(constDesc);
children1.add(constDesc2);
exprDesc=new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,udf,children1);
vc=new VectorizationContext("name",columns);
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterVarCharColumnBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterVarCharColumnNotBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(false)));
children1.set(1,new ExprNodeColumnDesc(Long.class,"col1","table",false));
children1.set(2,new ExprNodeConstantDesc(10));
children1.set(3,new ExprNodeConstantDesc(20));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterLongColumnBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterLongColumnNotBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(false)));
children1.set(1,new ExprNodeColumnDesc(Double.class,"col1","table",false));
children1.set(2,new ExprNodeConstantDesc(10.0d));
children1.set(3,new ExprNodeConstantDesc(20.0d));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterDoubleColumnBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterDoubleColumnNotBetween);
children1.set(0,new ExprNodeConstantDesc(new Boolean(false)));
children1.set(1,new ExprNodeColumnDesc(Timestamp.class,"col1","table",false));
children1.set(2,new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
children1.set(3,new ExprNodeConstantDesc("2013-11-06 00:00:00.000"));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterStringColumnBetween.class,ve.getClass());
children1.set(0,new ExprNodeConstantDesc(new Boolean(true)));
ve=vc.getVectorExpression(exprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterStringColumnNotBetween.class,ve.getClass());
}
InternalCallVerifier EqualityVerifier children=new ArrayList(2);
ExprNodeConstantDesc constDesc=new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo,20);
ExprNodeColumnDesc colDesc=new ExprNodeColumnDesc(Long.class,"a","table",false);
children.add(constDesc);
children.add(colDesc);
scalarGreaterColExpr.setChildren(children);
List columns=new ArrayList();
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(scalarGreaterColExpr,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterLongScalarGreaterLongColumn.class,ve.getClass());
}
InternalCallVerifier EqualityVerifier children=new ArrayList(2);
ExprNodeConstantDesc constDesc=new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo,20);
ExprNodeColumnDesc colDesc=new ExprNodeColumnDesc(Boolean.class,"a","table",false);
children.add(colDesc);
children.add(constDesc);
colEqualScalar.setChildren(children);
List columns=new ArrayList();
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(colEqualScalar,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(LongColEqualLongScalar.class,ve.getClass());
}
EqualityVerifier InternalCallVerifier EqualityVerifier children=new ArrayList(2);
ExprNodeConstantDesc constDesc=new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo,20);
ExprNodeColumnDesc colDesc=new ExprNodeColumnDesc(Long.class,"a","table",false);
children.add(constDesc);
children.add(colDesc);
ExprNodeGenericFuncDesc scalarMinusConstant=new ExprNodeGenericFuncDesc(TypeInfoFactory.longTypeInfo,gudf,children);
List columns=new ArrayList();
columns.add("a");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(scalarMinusConstant,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(),LongScalarSubtractLongColumn.class);
}
InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeGenericFuncDesc isNullExpr=new ExprNodeGenericFuncDesc();
isNullExpr.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
GenericUDFOPNull isNullUdf=new GenericUDFOPNull();
isNullExpr.setGenericUDF(isNullUdf);
List childOfIsNull=new ArrayList();
childOfIsNull.add(greaterExprDesc);
isNullExpr.setChildren(childOfIsNull);
List columns=new ArrayList();
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(isNullExpr,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(),SelectColumnIsNull.class);
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
assertEquals(2,ve.getChildExpressions()[0].getOutputColumn());
assertEquals(2,((SelectColumnIsNull)ve).getColNum());
ve=vc.getVectorExpression(isNullExpr,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(),IsNull.class);
assertEquals(2,((IsNull)ve).getColNum());
assertEquals(3,ve.getOutputColumn());
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
}
InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeGenericFuncDesc notExpr=new ExprNodeGenericFuncDesc();
notExpr.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
GenericUDFOPNot notUdf=new GenericUDFOPNot();
notExpr.setGenericUDF(notUdf);
List childOfNot=new ArrayList();
childOfNot.add(greaterExprDesc);
notExpr.setChildren(childOfNot);
List columns=new ArrayList();
columns.add("col0");
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(notExpr,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(),SelectColumnIsFalse.class);
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
ve=vc.getVectorExpression(notExpr,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(),NotCol.class);
assertEquals(ve.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
}
InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeColumnDesc col2Expr=new ExprNodeColumnDesc(Float.class,"col2","table",false);
ExprNodeConstantDesc const2Desc=new ExprNodeConstantDesc(new Float(1.0));
GenericUDFOPLessThan udf2=new GenericUDFOPLessThan();
ExprNodeGenericFuncDesc lessExprDesc=new ExprNodeGenericFuncDesc();
lessExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
lessExprDesc.setGenericUDF(udf2);
List children2=new ArrayList(2);
children2.add(col2Expr);
children2.add(const2Desc);
lessExprDesc.setChildren(children2);
GenericUDFOPAnd andUdf=new GenericUDFOPAnd();
ExprNodeGenericFuncDesc andExprDesc=new ExprNodeGenericFuncDesc();
andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
andExprDesc.setGenericUDF(andUdf);
List children3=new ArrayList(2);
children3.add(greaterExprDesc);
children3.add(lessExprDesc);
andExprDesc.setChildren(children3);
List columns=new ArrayList();
columns.add("col0");
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(andExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(),FilterExprAndExpr.class);
assertEquals(ve.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(ve.getChildExpressions()[1].getClass(),FilterDoubleColLessDoubleScalar.class);
GenericUDFOPOr orUdf=new GenericUDFOPOr();
ExprNodeGenericFuncDesc orExprDesc=new ExprNodeGenericFuncDesc();
orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
orExprDesc.setGenericUDF(orUdf);
List children4=new ArrayList(2);
children4.add(greaterExprDesc);
children4.add(lessExprDesc);
orExprDesc.setChildren(children4);
VectorExpression veOr=vc.getVectorExpression(orExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veOr.getClass(),FilterExprOrExpr.class);
assertEquals(veOr.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(veOr.getChildExpressions()[1].getClass(),FilterDoubleColLessDoubleScalar.class);
}
InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeColumnDesc col2Expr=new ExprNodeColumnDesc(Float.class,"col2","table",false);
ExprNodeConstantDesc const2Desc=new ExprNodeConstantDesc(new Float(1.0));
GenericUDFOPLessThan udf2=new GenericUDFOPLessThan();
ExprNodeGenericFuncDesc lessExprDesc=new ExprNodeGenericFuncDesc();
lessExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
lessExprDesc.setGenericUDF(udf2);
List children2=new ArrayList(2);
children2.add(col2Expr);
children2.add(const2Desc);
lessExprDesc.setChildren(children2);
ExprNodeColumnDesc col3Expr=new ExprNodeColumnDesc(Integer.class,"col3","table",false);
ExprNodeConstantDesc const3Desc=new ExprNodeConstantDesc(new Integer(10));
GenericUDFOPGreaterThan udf3=new GenericUDFOPGreaterThan();
ExprNodeGenericFuncDesc greaterExprDesc3=new ExprNodeGenericFuncDesc();
greaterExprDesc3.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
greaterExprDesc3.setGenericUDF(udf3);
List children3=new ArrayList(2);
children3.add(col3Expr);
children3.add(const3Desc);
greaterExprDesc3.setChildren(children3);
GenericUDFOPAnd andUdf=new GenericUDFOPAnd();
ExprNodeGenericFuncDesc andExprDesc=new ExprNodeGenericFuncDesc();
andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
andExprDesc.setGenericUDF(andUdf);
List children4=new ArrayList(2);
children4.add(greaterExprDesc);
children4.add(lessExprDesc);
children4.add(greaterExprDesc3);
andExprDesc.setChildren(children4);
List columns=new ArrayList();
columns.add("col0");
columns.add("col1");
columns.add("col2");
columns.add("col3");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression ve=vc.getVectorExpression(andExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(ve.getClass(),FilterExprAndExpr.class);
assertEquals(ve.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(ve.getChildExpressions()[1].getClass(),FilterDoubleColLessDoubleScalar.class);
assertEquals(ve.getChildExpressions()[2].getClass(),FilterLongColGreaterLongScalar.class);
GenericUDFOPOr orUdf=new GenericUDFOPOr();
ExprNodeGenericFuncDesc orExprDesc=new ExprNodeGenericFuncDesc();
orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
orExprDesc.setGenericUDF(orUdf);
List children5=new ArrayList(2);
children5.add(greaterExprDesc);
children5.add(lessExprDesc);
children5.add(greaterExprDesc3);
orExprDesc.setChildren(children5);
VectorExpression veOr=vc.getVectorExpression(orExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veOr.getClass(),FilterExprOrExpr.class);
assertEquals(veOr.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(veOr.getChildExpressions()[1].getClass(),FilterDoubleColLessDoubleScalar.class);
assertEquals(ve.getChildExpressions()[2].getClass(),FilterLongColGreaterLongScalar.class);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier children1=new ArrayList(2);
List children2=new ArrayList(2);
List children3=new ArrayList(2);
List children4=new ArrayList(2);
List children5=new ArrayList(2);
children1.add(minusExpr);
children1.add(multiplyExpr);
sumExpr.setChildren(children1);
children2.add(sum2Expr);
children2.add(col3Expr);
minusExpr.setChildren(children2);
children3.add(col1Expr);
children3.add(col2Expr);
sum2Expr.setChildren(children3);
children4.add(col4Expr);
children4.add(modExpr);
multiplyExpr.setChildren(children4);
children5.add(col5Expr);
children5.add(col6Expr);
modExpr.setChildren(children5);
VectorizationContext vc=new VectorizationContext("name");
vc.addInitialColumn("col1");
vc.addInitialColumn("col2");
vc.addInitialColumn("col3");
vc.addInitialColumn("col4");
vc.addInitialColumn("col5");
vc.addInitialColumn("col6");
vc.finishedAddingInitialColumns();
VectorExpression ve=vc.getVectorExpression(sumExpr,VectorExpressionDescriptor.Mode.PROJECTION);
assertTrue(ve instanceof LongColAddLongColumn);
assertEquals(2,ve.getChildExpressions().length);
VectorExpression childExpr1=ve.getChildExpressions()[0];
VectorExpression childExpr2=ve.getChildExpressions()[1];
System.out.println(ve.toString());
assertEquals(6,ve.getOutputColumn());
assertTrue(childExpr1 instanceof LongColSubtractLongColumn);
assertEquals(1,childExpr1.getChildExpressions().length);
assertTrue(childExpr1.getChildExpressions()[0] instanceof LongColAddLongColumn);
assertEquals(7,childExpr1.getOutputColumn());
assertEquals(6,childExpr1.getChildExpressions()[0].getOutputColumn());
assertTrue(childExpr2 instanceof LongColMultiplyLongColumn);
assertEquals(1,childExpr2.getChildExpressions().length);
assertTrue(childExpr2.getChildExpressions()[0] instanceof LongColModuloLongColumn);
assertEquals(8,childExpr2.getOutputColumn());
assertEquals(6,childExpr2.getChildExpressions()[0].getOutputColumn());
}
EqualityVerifier InternalCallVerifier EqualityVerifier children1=new ArrayList(2);
children1.add(col1Expr);
children1.add(constDesc);
greaterExprDesc.setChildren(children1);
ExprNodeColumnDesc col2Expr=new ExprNodeColumnDesc(Boolean.class,"col2","table",false);
GenericUDFOPAnd andUdf=new GenericUDFOPAnd();
ExprNodeGenericFuncDesc andExprDesc=new ExprNodeGenericFuncDesc();
andExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
andExprDesc.setGenericUDF(andUdf);
List children3=new ArrayList(2);
children3.add(greaterExprDesc);
children3.add(col2Expr);
andExprDesc.setChildren(children3);
List columns=new ArrayList();
columns.add("col1");
columns.add("col2");
VectorizationContext vc=new VectorizationContext("name",columns);
VectorExpression veAnd=vc.getVectorExpression(andExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veAnd.getClass(),FilterExprAndExpr.class);
assertEquals(veAnd.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(veAnd.getChildExpressions()[1].getClass(),SelectColumnIsTrue.class);
veAnd=vc.getVectorExpression(andExprDesc,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(veAnd.getClass(),ColAndCol.class);
assertEquals(1,veAnd.getChildExpressions().length);
assertEquals(veAnd.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
assertEquals(2,((ColAndCol)veAnd).getColNum1());
assertEquals(1,((ColAndCol)veAnd).getColNum2());
assertEquals(3,((ColAndCol)veAnd).getOutputColumn());
GenericUDFOPOr orUdf=new GenericUDFOPOr();
ExprNodeGenericFuncDesc orExprDesc=new ExprNodeGenericFuncDesc();
orExprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
orExprDesc.setGenericUDF(orUdf);
List children4=new ArrayList(2);
children4.add(greaterExprDesc);
children4.add(col2Expr);
orExprDesc.setChildren(children4);
vc=new VectorizationContext("name",columns);
VectorExpression veOr=vc.getVectorExpression(orExprDesc,VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veOr.getClass(),FilterExprOrExpr.class);
assertEquals(veOr.getChildExpressions()[0].getClass(),FilterLongColGreaterLongScalar.class);
assertEquals(veOr.getChildExpressions()[1].getClass(),SelectColumnIsTrue.class);
veOr=vc.getVectorExpression(orExprDesc,VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(veOr.getClass(),ColOrCol.class);
assertEquals(1,veAnd.getChildExpressions().length);
assertEquals(veAnd.getChildExpressions()[0].getClass(),LongColGreaterLongScalar.class);
assertEquals(2,((ColOrCol)veOr).getColNum1());
assertEquals(1,((ColOrCol)veOr).getColNum2());
assertEquals(3,((ColOrCol)veOr).getOutputColumn());
}
Class: org.apache.hadoop.hive.ql.exec.vector.TestVectorizedRowBatch EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestConstantVectorExpression APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestDecimalUtil EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestUnaryMinus IterativeVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorArithmeticExpressions BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier IterativeVerifier BranchVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorConditionalExpressions EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorExpressionWriters EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorFilterExpressions APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorGenericDateExpressions EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorLogicalExpressions EqualityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier IterativeVerifier EqualityVerifier IterativeVerifier EqualityVerifier EqualityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorMathFunctions EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorScalarColArithmetic IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier IterativeVerifier BranchVerifier BooleanVerifier EqualityVerifier HybridVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorStringExpressions EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorTimestampExpressions EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.expressions.TestVectorTypeCasts EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hadoop.hive.ql.exec.vector.udf.TestVectorUDFAdaptor APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier children=new ArrayList();
children.add(new ExprNodeColumnDesc(typeInfoStr,"col0","tablename",false));
children.add(new ExprNodeConstantDesc(typeInfoStr,"UNKNOWN"));
VectorUDFArgDesc[] argDescs=new VectorUDFArgDesc[2];
for (int i=0; i < 2; i++) {
argDescs[i]=new VectorUDFArgDesc();
}
argDescs[0].setVariable(0);
argDescs[1].setConstant((ExprNodeConstantDesc)children.get(1));
funcDesc=new ExprNodeGenericFuncDesc(typeInfoStr,genericUDF,"myisnull",children);
VectorUDFAdaptor vudf=null;
try {
vudf=new VectorUDFAdaptor(funcDesc,3,"String",argDescs);
}
catch ( HiveException e) {
assertTrue(false);
}
VectorizedRowBatch b;
byte[] red=null;
byte[] unknown=null;
try {
red="red".getBytes("UTF-8");
unknown="UNKNOWN".getBytes("UTF-8");
}
catch ( Exception e) {
;
}
BytesColumnVector out;
b=getBatchStrDblLongWithStrOut();
b.cols[0].noNulls=false;
b.cols[0].isNull[0]=true;
vudf.evaluate(b);
out=(BytesColumnVector)b.cols[3];
int cmp=StringExpr.compare(red,0,red.length,out.vector[1],out.start[1],out.length[1]);
assertEquals(0,cmp);
cmp=StringExpr.compare(unknown,0,unknown.length,out.vector[0],out.start[0],out.length[0]);
assertEquals(0,cmp);
assertTrue(out.noNulls || !out.isNull[0]);
}
APIUtilityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier children=new ArrayList();
children.add(new ExprNodeColumnDesc(typeInfoStr,"col0","tablename",false));
children.add(new ExprNodeColumnDesc(typeInfoLong,"col1","tablename",false));
children.add(new ExprNodeColumnDesc(typeInfoDbl,"col2","tablename",false));
VectorUDFArgDesc[] argDescs=new VectorUDFArgDesc[3];
for (int i=0; i < 3; i++) {
argDescs[i]=new VectorUDFArgDesc();
argDescs[i].setVariable(i);
}
funcDesc=new ExprNodeGenericFuncDesc(typeInfoStr,genericUDFBridge,genericUDFBridge.getUdfName(),children);
VectorUDFAdaptor vudf=null;
try {
vudf=new VectorUDFAdaptor(funcDesc,3,"String",argDescs);
}
catch ( HiveException e) {
assertTrue(false);
throw new RuntimeException(e);
}
VectorizedRowBatch b=getBatchStrDblLongWithStrOut();
vudf.evaluate(b);
byte[] result=null;
byte[] result2=null;
try {
result="red:1:1.0".getBytes("UTF-8");
result2="blue:0:0.0".getBytes("UTF-8");
}
catch ( Exception e) {
;
}
BytesColumnVector out=(BytesColumnVector)b.cols[3];
int cmp=StringExpr.compare(result,0,result.length,out.vector[1],out.start[1],out.length[1]);
assertEquals(0,cmp);
assertTrue(out.noNulls);
b=getBatchStrDblLongWithStrOut();
b.cols[1].noNulls=false;
vudf.evaluate(b);
out=(BytesColumnVector)b.cols[3];
assertFalse(out.noNulls);
assertTrue(out.isNull[1]);
b=getBatchStrDblLongWithStrOut();
b.cols[0].isRepeating=true;
b.cols[1].isRepeating=true;
b.cols[2].isRepeating=true;
vudf.evaluate(b);
out=(BytesColumnVector)b.cols[3];
assertTrue(out.isRepeating);
cmp=StringExpr.compare(result2,0,result2.length,out.vector[0],out.start[0],out.length[0]);
assertEquals(0,cmp);
assertTrue(out.noNulls);
}
BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier children=new ArrayList();
ExprNodeColumnDesc colDesc=new ExprNodeColumnDesc(typeInfo,"col0","tablename",false);
children.add(colDesc);
VectorUDFArgDesc[] argDescs=new VectorUDFArgDesc[1];
argDescs[0]=new VectorUDFArgDesc();
argDescs[0].setVariable(0);
funcDesc=new ExprNodeGenericFuncDesc(typeInfo,genericUDFBridge,genericUDFBridge.getUdfName(),children);
VectorUDFAdaptor vudf=null;
try {
vudf=new VectorUDFAdaptor(funcDesc,1,"Long",argDescs);
}
catch ( HiveException e) {
assertTrue(false);
}
VectorizedRowBatch b=getBatchLongInLongOut();
vudf.evaluate(b);
LongColumnVector out=(LongColumnVector)b.cols[1];
assertEquals(1000,out.vector[0]);
assertEquals(1001,out.vector[1]);
assertEquals(1002,out.vector[2]);
assertTrue(out.noNulls);
assertFalse(out.isRepeating);
b=getBatchLongInLongOut();
out=(LongColumnVector)b.cols[1];
b.cols[0].noNulls=false;
vudf.evaluate(b);
assertFalse(out.noNulls);
assertEquals(1000,out.vector[0]);
assertEquals(1001,out.vector[1]);
assertTrue(out.isNull[2]);
assertFalse(out.isRepeating);
b=getBatchLongInLongOut();
out=(LongColumnVector)b.cols[1];
b.cols[0].isRepeating=true;
vudf.evaluate(b);
assertTrue(b.cols[1].isRepeating && out.vector[0] == 1000 || !b.cols[1].isRepeating && out.vector[2] == 1000);
assertEquals(3,b.size);
}
Class: org.apache.hadoop.hive.ql.hooks.TestHooks APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.TestAcidUtils APIUtilityVerifier InternalCallVerifier EqualityVerifier delts=dir.getCurrentDirectories();
assertEquals(1,delts.size());
assertEquals("mock:/tbl/part1/delta_1_1",delts.get(0).getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier result=dir.getOriginalFiles();
assertEquals(5,result.size());
assertEquals("mock:/tbl/part1/000000_0",result.get(0).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/000001_1",result.get(1).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/000002_0",result.get(2).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/random",result.get(3).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/subdir/000000_0",result.get(4).getFileStatus().getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier obsolete=dir.getObsolete();
assertEquals(5,obsolete.size());
assertEquals("mock:/tbl/part1/base_10",obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/base_5",obsolete.get(1).getPath().toString());
assertEquals("mock:/tbl/part1/delta_025_030",obsolete.get(2).getPath().toString());
assertEquals("mock:/tbl/part1/delta_025_025",obsolete.get(3).getPath().toString());
assertEquals("mock:/tbl/part1/delta_029_029",obsolete.get(4).getPath().toString());
assertEquals(0,dir.getOriginalFiles().size());
List deltas=dir.getCurrentDirectories();
assertEquals(1,deltas.size());
AcidUtils.ParsedDelta delt=deltas.get(0);
assertEquals("mock:/tbl/part1/delta_050_105",delt.getPath().toString());
assertEquals(50,delt.getMinTransaction());
assertEquals(105,delt.getMaxTransaction());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier delts=dir.getCurrentDirectories();
assertEquals(2,delts.size());
assertEquals("mock:/tbl/part1/delta_1_1",delts.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_2_5",delts.get(1).getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier obsolete=dir.getObsolete();
assertEquals(2,obsolete.size());
assertEquals("mock:/tbl/part1/delta_025_025",obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_029_029",obsolete.get(1).getPath().toString());
List result=dir.getOriginalFiles();
assertEquals(5,result.size());
assertEquals("mock:/tbl/part1/000000_0",result.get(0).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/000001_1",result.get(1).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/000002_0",result.get(2).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/random",result.get(3).getFileStatus().getPath().toString());
assertEquals("mock:/tbl/part1/subdir/000000_0",result.get(4).getFileStatus().getPath().toString());
List deltas=dir.getCurrentDirectories();
assertEquals(2,deltas.size());
AcidUtils.ParsedDelta delt=deltas.get(0);
assertEquals("mock:/tbl/part1/delta_025_030",delt.getPath().toString());
assertEquals(25,delt.getMinTransaction());
assertEquals(30,delt.getMaxTransaction());
delt=deltas.get(1);
assertEquals("mock:/tbl/part1/delta_050_100",delt.getPath().toString());
assertEquals(50,delt.getMinTransaction());
assertEquals(100,delt.getMaxTransaction());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier obsolete=dir.getObsolete();
assertEquals(3,obsolete.size());
assertEquals("mock:/tbl/part1/base_5",obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/base_10",dir.getBaseDirectory().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier obsoletes=dir.getObsolete();
assertEquals(4,obsoletes.size());
assertEquals("mock:/tbl/part1/base_10",obsoletes.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/base_100",obsoletes.get(1).getPath().toString());
assertEquals("mock:/tbl/part1/base_25",obsoletes.get(2).getPath().toString());
assertEquals("mock:/tbl/part1/base_5",obsoletes.get(3).getPath().toString());
assertEquals(0,dir.getOriginalFiles().size());
assertEquals(0,dir.getCurrentDirectories().size());
dir=AcidUtils.getAcidState(part,conf,new ValidReadTxnList("10:"));
assertEquals("mock:/tbl/part1/base_200",dir.getBaseDirectory().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier delts=dir.getCurrentDirectories();
assertEquals(2,delts.size());
assertEquals("mock:/tbl/part1/delta_1_1",delts.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_2_5",delts.get(1).getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier obsolete=dir.getObsolete();
assertEquals(5,obsolete.size());
assertEquals("mock:/tbl/part1/delta_052_55",obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_058_58",obsolete.get(1).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0060_60_1",obsolete.get(2).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0060_60_4",obsolete.get(3).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0060_60_7",obsolete.get(4).getPath().toString());
List delts=dir.getCurrentDirectories();
assertEquals(5,delts.size());
assertEquals("mock:/tbl/part1/delta_40_60",delts.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_00061_61_0",delts.get(1).getPath().toString());
assertEquals("mock:/tbl/part1/delta_000062_62_0",delts.get(2).getPath().toString());
assertEquals("mock:/tbl/part1/delta_000062_62_3",delts.get(3).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0000063_63_0",delts.get(4).getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier obsolete=dir.getObsolete();
assertEquals(2,obsolete.size());
assertEquals("mock:/tbl/part1/delta_052_55",obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0060_60",obsolete.get(1).getPath().toString());
List delts=dir.getCurrentDirectories();
assertEquals(4,delts.size());
assertEquals("mock:/tbl/part1/delta_40_60",delts.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_00061_61",delts.get(1).getPath().toString());
assertEquals("mock:/tbl/part1/delta_000062_62",delts.get(2).getPath().toString());
assertEquals("mock:/tbl/part1/delta_0000063_63",delts.get(3).getPath().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier delts=dir.getCurrentDirectories();
assertEquals(1,delts.size());
assertEquals("mock:/tbl/part1/delta_1_1",delts.get(0).getPath().toString());
}
Class: org.apache.hadoop.hive.ql.io.TestHiveInputOutputBuffer InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.TestRCFile APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.TestStorageFormatDescriptor EqualityVerifier Class: org.apache.hadoop.hive.ql.io.avro.TestAvroGenericRecordReader InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestColumnStatistics APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier stripeStats=reader.getStripeStatistics();
StripeStatistics ss1=stripeStats.get(0);
ColumnStatistics ss1_cs1=ss1.getColumnStatistics()[0];
ColumnStatistics ss1_cs2=ss1.getColumnStatistics()[1];
ColumnStatistics ss1_cs3=ss1.getColumnStatistics()[2];
assertEquals(false,ss1_cs1.hasNull());
assertEquals(false,ss1_cs2.hasNull());
assertEquals(true,ss1_cs3.hasNull());
StripeStatistics ss2=stripeStats.get(1);
ColumnStatistics ss2_cs1=ss2.getColumnStatistics()[0];
ColumnStatistics ss2_cs2=ss2.getColumnStatistics()[1];
ColumnStatistics ss2_cs3=ss2.getColumnStatistics()[2];
assertEquals(false,ss2_cs1.hasNull());
assertEquals(false,ss2_cs2.hasNull());
assertEquals(true,ss2_cs3.hasNull());
StripeStatistics ss3=stripeStats.get(2);
ColumnStatistics ss3_cs1=ss3.getColumnStatistics()[0];
ColumnStatistics ss3_cs2=ss3.getColumnStatistics()[1];
ColumnStatistics ss3_cs3=ss3.getColumnStatistics()[2];
assertEquals(false,ss3_cs1.hasNull());
assertEquals(false,ss3_cs2.hasNull());
assertEquals(false,ss3_cs3.hasNull());
StripeStatistics ss4=stripeStats.get(3);
ColumnStatistics ss4_cs1=ss4.getColumnStatistics()[0];
ColumnStatistics ss4_cs2=ss4.getColumnStatistics()[1];
ColumnStatistics ss4_cs3=ss4.getColumnStatistics()[2];
assertEquals(false,ss4_cs1.hasNull());
assertEquals(false,ss4_cs2.hasNull());
assertEquals(true,ss4_cs3.hasNull());
PrintStream origOut=System.out;
String outputFilename="orc-file-has-null.out";
FileOutputStream myOut=new FileOutputStream(workDir + File.separator + outputFilename);
System.setOut(new PrintStream(myOut));
FileDump.main(new String[]{testFilePath.toString(),"--rowindex=2"});
System.out.flush();
System.setOut(origOut);
TestFileDump.checkOutput(outputFilename,workDir + File.separator + outputFilename);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestFileDump APIUtilityVerifier EqualityVerifier m=new HashMap(2);
m.put("k1","v1");
writer.addRow(new AllTypesRecord(true,(byte)10,(short)100,1000,10000L,4.0f,20.0,HiveDecimal.create("4.2222"),new Timestamp(1416967764000L),new Date(1416967764000L),"string",new HiveChar("hello",5),new HiveVarchar("hello",10),m,Arrays.asList(100,200),new AllTypesRecord.Struct(10,"foo")));
m.clear();
m.put("k3","v3");
writer.addRow(new AllTypesRecord(false,(byte)20,(short)200,2000,20000L,8.0f,40.0,HiveDecimal.create("2.2222"),new Timestamp(1416967364000L),new Date(1411967764000L),"abcd",new HiveChar("world",5),new HiveVarchar("world",10),m,Arrays.asList(200,300),new AllTypesRecord.Struct(20,"bar")));
writer.close();
PrintStream origOut=System.out;
ByteArrayOutputStream myOut=new ByteArrayOutputStream();
System.setOut(new PrintStream(myOut));
FileDump.main(new String[]{testFilePath.toString(),"-d"});
System.out.flush();
System.setOut(origOut);
String[] lines=myOut.toString().split("\n");
assertEquals("{\"b\":true,\"bt\":10,\"s\":100,\"i\":1000,\"l\":10000,\"f\":4,\"d\":20,\"de\":\"4.2222\",\"t\":\"2014-11-25 18:09:24\",\"dt\":\"2014-11-25\",\"str\":\"string\",\"c\":\"hello \",\"vc\":\"hello\",\"m\":[{\"_key\":\"k1\",\"_value\":\"v1\"}],\"a\":[100,200],\"st\":{\"i\":10,\"s\":\"foo\"}}",lines[0]);
assertEquals("{\"b\":false,\"bt\":20,\"s\":200,\"i\":2000,\"l\":20000,\"f\":8,\"d\":40,\"de\":\"2.2222\",\"t\":\"2014-11-25 18:02:44\",\"dt\":\"2014-09-28\",\"str\":\"abcd\",\"c\":\"world \",\"vc\":\"world\",\"m\":[{\"_key\":\"k3\",\"_value\":\"v3\"}],\"a\":[200,300],\"st\":{\"i\":20,\"s\":\"bar\"}}",lines[1]);
}
Class: org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier (),true,null,null));
List results=splitter.call();
OrcSplit result=results.get(0);
assertEquals(3,results.size());
assertEquals(3,result.getStart());
assertEquals(400,result.getLength());
assertEquals(167468,result.getProjectedColumnsUncompressedSize());
result=results.get(1);
assertEquals(403,result.getStart());
assertEquals(400,result.getLength());
assertEquals(167468,result.getProjectedColumnsUncompressedSize());
result=results.get(2);
assertEquals(803,result.getStart());
assertEquals(100,result.getLength());
assertEquals(41867,result.getProjectedColumnsUncompressedSize());
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMAXSPLITSIZE,0);
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMINSPLITSIZE,0);
context=new OrcInputFormat.Context(conf);
splitter=new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context,fs,fs.getFileStatus(new Path("/a/file")),null,true,new ArrayList(),true,null,null));
results=splitter.call();
assertEquals(5,results.size());
for (int i=0; i < stripeSizes.length; ++i) {
assertEquals("checking stripe " + i + " size",stripeSizes[i],results.get(i).getLength());
if (i == stripeSizes.length - 1) {
assertEquals(41867,results.get(i).getProjectedColumnsUncompressedSize());
}
else {
assertEquals(83734,results.get(i).getProjectedColumnsUncompressedSize());
}
}
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMAXSPLITSIZE,1000);
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMINSPLITSIZE,100000);
context=new OrcInputFormat.Context(conf);
splitter=new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context,fs,fs.getFileStatus(new Path("/a/file")),null,true,new ArrayList(),true,null,null));
results=splitter.call();
assertEquals(1,results.size());
result=results.get(0);
assertEquals(3,result.getStart());
assertEquals(900,result.getLength());
assertEquals(376804,result.getProjectedColumnsUncompressedSize());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier ();
InputSplit[] splits=inputFormat.getSplits(conf,10);
assertEquals(1,splits.length);
conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS,BigRow.getColumnNamesProperty());
conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES,BigRow.getColumnTypesProperty());
HiveConf.setBoolVar(conf,HiveConf.ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN,true);
org.apache.hadoop.mapred.RecordReader reader=inputFormat.getRecordReader(splits[0],conf,Reporter.NULL);
NullWritable key=reader.createKey();
VectorizedRowBatch value=reader.createValue();
assertEquals(true,reader.next(key,value));
assertEquals(100,value.count());
LongColumnVector booleanColumn=(LongColumnVector)value.cols[0];
LongColumnVector byteColumn=(LongColumnVector)value.cols[1];
LongColumnVector shortColumn=(LongColumnVector)value.cols[2];
LongColumnVector intColumn=(LongColumnVector)value.cols[3];
LongColumnVector longColumn=(LongColumnVector)value.cols[4];
DoubleColumnVector floatColumn=(DoubleColumnVector)value.cols[5];
DoubleColumnVector doubleCoulmn=(DoubleColumnVector)value.cols[6];
BytesColumnVector stringColumn=(BytesColumnVector)value.cols[7];
DecimalColumnVector decimalColumn=(DecimalColumnVector)value.cols[8];
LongColumnVector dateColumn=(LongColumnVector)value.cols[9];
LongColumnVector timestampColumn=(LongColumnVector)value.cols[10];
for (int i=0; i < 100; i++) {
assertEquals("checking boolean " + i,i % 2 == 0 ? 1 : 0,booleanColumn.vector[i]);
assertEquals("checking byte " + i,(byte)i,byteColumn.vector[i]);
assertEquals("checking short " + i,(short)i,shortColumn.vector[i]);
assertEquals("checking int " + i,i,intColumn.vector[i]);
assertEquals("checking long " + i,i,longColumn.vector[i]);
assertEquals("checking float " + i,i,floatColumn.vector[i],0.0001);
assertEquals("checking double " + i,i,doubleCoulmn.vector[i],0.0001);
Text strValue=new Text();
strValue.set(stringColumn.vector[i],stringColumn.start[i],stringColumn.length[i]);
assertEquals("checking string " + i,new Text(Long.toHexString(i)),strValue);
assertEquals("checking decimal " + i,HiveDecimal.create(i),decimalColumn.vector[i].getHiveDecimal());
assertEquals("checking date " + i,i,dateColumn.vector[i]);
long millis=(long)i * MILLIS_IN_DAY;
millis-=LOCAL_TIMEZONE.getOffset(millis);
assertEquals("checking timestamp " + i,millis * 1000000L,timestampColumn.vector[i]);
}
assertEquals(false,reader.next(key,value));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier ",inspector.getTypeName());
InputFormat in=new OrcInputFormat();
FileInputFormat.setInputPaths(conf,testFilePath.toString());
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
ArrayList fileList=new ArrayList();
assertEquals(false,((InputFormatChecker)in).validateInput(fs,new HiveConf(),fileList));
fileList.add(fs.getFileStatus(testFilePath));
assertEquals(true,((InputFormatChecker)in).validateInput(fs,new HiveConf(),fileList));
fileList.add(fs.getFileStatus(workDir));
assertEquals(false,((InputFormatChecker)in).validateInput(fs,new HiveConf(),fileList));
conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS,MyRow.getColumnNamesProperty());
conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES,MyRow.getColumnTypesProperty());
org.apache.hadoop.mapred.RecordReader reader=in.getRecordReader(splits[0],conf,Reporter.NULL);
Object key=reader.createKey();
Writable value=(Writable)reader.createValue();
int rowNum=0;
List fields=inspector.getAllStructFieldRefs();
IntObjectInspector intInspector=(IntObjectInspector)fields.get(0).getFieldObjectInspector();
while (reader.next(key,value)) {
assertEquals(++rowNum,intInspector.get(inspector.getStructFieldData(serde.deserialize(value),fields.get(0))));
assertEquals(2,intInspector.get(inspector.getStructFieldData(serde.deserialize(value),fields.get(1))));
}
assertEquals(3,rowNum);
assertEquals(1.0,reader.getProgress(),0.00001);
reader.close();
ColumnProjectionUtils.appendReadColumns(conf,Collections.singletonList(0));
reader=in.getRecordReader(splits[0],conf,Reporter.NULL);
key=reader.createKey();
value=(Writable)reader.createValue();
rowNum=0;
fields=inspector.getAllStructFieldRefs();
while (reader.next(key,value)) {
assertEquals(++rowNum,intInspector.get(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(null,inspector.getStructFieldData(value,fields.get(1)));
}
assertEquals(3,rowNum);
reader.close();
ColumnProjectionUtils.setReadAllColumns(conf);
reader=in.getRecordReader(splits[0],conf,Reporter.NULL);
key=reader.createKey();
value=(Writable)reader.createValue();
rowNum=0;
fields=inspector.getAllStructFieldRefs();
while (reader.next(key,value)) {
assertEquals(++rowNum,intInspector.get(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(2,intInspector.get(inspector.getStructFieldData(serde.deserialize(value),fields.get(1))));
}
assertEquals(3,rowNum);
reader.close();
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier ");
SerDeUtils.initializeSerDe(serde,conf,properties,null);
inspector=(StructObjectInspector)serde.getObjectInspector();
InputFormat in=new OrcInputFormat();
FileInputFormat.setInputPaths(conf,testFilePath.toString());
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
ColumnProjectionUtils.appendReadColumns(conf,Collections.singletonList(1));
conf.set("columns","z,r");
conf.set("columns.types","int:struct");
org.apache.hadoop.mapred.RecordReader reader=in.getRecordReader(splits[0],conf,Reporter.NULL);
Object key=reader.createKey();
Object value=reader.createValue();
int rowNum=0;
List fields=inspector.getAllStructFieldRefs();
StructObjectInspector inner=(StructObjectInspector)fields.get(1).getFieldObjectInspector();
List inFields=inner.getAllStructFieldRefs();
IntObjectInspector intInspector=(IntObjectInspector)fields.get(0).getFieldObjectInspector();
while (reader.next(key,value)) {
assertEquals(null,inspector.getStructFieldData(value,fields.get(0)));
Object sub=inspector.getStructFieldData(value,fields.get(1));
assertEquals(3 * rowNum + 1,intInspector.get(inner.getStructFieldData(sub,inFields.get(0))));
assertEquals(3 * rowNum + 2,intInspector.get(inner.getStructFieldData(sub,inFields.get(1))));
rowNum+=1;
}
assertEquals(3,rowNum);
reader.close();
}
InternalCallVerifier EqualityVerifier ");
StructObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector=(StructObjectInspector)ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
SerDe serde=new OrcSerde();
OutputFormat outFormat=new OrcOutputFormat();
conf.setInt("mapred.max.split.size",50);
RecordWriter writer=outFormat.getRecordWriter(fs,conf,testFilePath.toString(),Reporter.NULL);
writer.write(NullWritable.get(),serde.serialize(new NestedRow(1,2,3),inspector));
writer.write(NullWritable.get(),serde.serialize(new NestedRow(4,5,6),inspector));
writer.write(NullWritable.get(),serde.serialize(new NestedRow(7,8,9),inspector));
writer.close(Reporter.NULL);
serde=new OrcSerde();
SearchArgument sarg=SearchArgumentFactory.newBuilder().startAnd().lessThan("z",PredicateLeaf.Type.LONG,new Long(0)).end().build();
conf.set("sarg.pushdown",toKryo(sarg));
conf.set("hive.io.file.readcolumn.names","z,r");
SerDeUtils.initializeSerDe(serde,conf,properties,null);
inspector=(StructObjectInspector)serde.getObjectInspector();
InputFormat in=new OrcInputFormat();
FileInputFormat.setInputPaths(conf,testFilePath.toString());
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(0,splits.length);
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier ();
InputSplit[] splits=inputFormat.getSplits(conf,10);
assertEquals(1,splits.length);
org.apache.hadoop.mapred.RecordReader reader=inputFormat.getRecordReader(splits[0],conf,Reporter.NULL);
NullWritable key=reader.createKey();
VectorizedRowBatch value=reader.createValue();
assertEquals(true,reader.next(key,value));
assertEquals(10,value.count());
LongColumnVector col0=(LongColumnVector)value.cols[0];
for (int i=0; i < 10; i++) {
assertEquals("checking " + i,i,col0.vector[i]);
}
assertEquals(false,reader.next(key,value));
}
InternalCallVerifier EqualityVerifier types=new ArrayList();
OrcProto.Type.Builder builder=OrcProto.Type.newBuilder();
builder.setKind(OrcProto.Type.Kind.STRUCT).addAllFieldNames(Arrays.asList("op","otid","bucket","rowid","ctid","row")).addAllSubtypes(Arrays.asList(1,2,3,4,5,6));
types.add(builder.build());
builder.clear().setKind(OrcProto.Type.Kind.INT);
types.add(builder.build());
types.add(builder.build());
types.add(builder.build());
types.add(builder.build());
types.add(builder.build());
builder.clear().setKind(OrcProto.Type.Kind.STRUCT).addAllFieldNames(Arrays.asList("url","purchase","cost","store")).addAllSubtypes(Arrays.asList(7,8,9,10));
types.add(builder.build());
builder.clear().setKind(OrcProto.Type.Kind.STRING);
types.add(builder.build());
builder.clear().setKind(OrcProto.Type.Kind.INT);
types.add(builder.build());
types.add(builder.build());
types.add(builder.build());
SearchArgument isNull=SearchArgumentFactory.newBuilder().startAnd().isNull("cost",PredicateLeaf.Type.LONG).end().build();
conf.set(ConvertAstToSearchArg.SARG_PUSHDOWN,toKryo(isNull));
conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR,"url,cost");
options.include(new boolean[]{true,true,false,true,false});
OrcInputFormat.setSearchArgument(options,types,conf,false);
String[] colNames=options.getColumnNames();
assertEquals(null,colNames[0]);
assertEquals("url",colNames[1]);
assertEquals(null,colNames[2]);
assertEquals("cost",colNames[3]);
assertEquals(null,colNames[4]);
SearchArgument arg=options.getSearchArgument();
List leaves=arg.getLeaves();
assertEquals("cost",leaves.get(0).getColumnName());
assertEquals(PredicateLeaf.Operator.IS_NULL,leaves.get(0).getOperator());
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier ();
InputSplit[] splits=inputFormat.getSplits(conf,1);
assertEquals(3,splits.length);
HiveInputFormat.HiveInputSplit split=(HiveInputFormat.HiveInputSplit)splits[0];
assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",split.inputFormatClassName());
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000",split.getPath().toString());
assertEquals(0,split.getStart());
assertEquals(607,split.getLength());
split=(HiveInputFormat.HiveInputSplit)splits[1];
assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",split.inputFormatClassName());
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001",split.getPath().toString());
assertEquals(0,split.getStart());
assertEquals(629,split.getLength());
CombineHiveInputFormat.CombineHiveInputSplit combineSplit=(CombineHiveInputFormat.CombineHiveInputSplit)splits[2];
assertEquals(BUCKETS,combineSplit.getNumPaths());
for (int bucket=0; bucket < BUCKETS; ++bucket) {
assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0",combineSplit.getPath(bucket).toString());
assertEquals(0,combineSplit.getOffset(bucket));
assertEquals(241,combineSplit.getLength(bucket));
}
String[] hosts=combineSplit.getLocations();
assertEquals(2,hosts.length);
}
EqualityVerifier ExceptionVerifier HybridVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier ();
InputSplit[] splits=inputFormat.getSplits(conf,1);
assertEquals(1,splits.length);
CombineHiveInputFormat.CombineHiveInputSplit split=(CombineHiveInputFormat.CombineHiveInputSplit)splits[0];
assertEquals(2,split.getNumPaths());
assertEquals(partDir.toString() + "/0_0",split.getPath(0).toString());
assertEquals(partDir.toString() + "/1_0",split.getPath(1).toString());
assertEquals(length0,split.getLength(0));
assertEquals(outputStream.file.length,split.getLength(1));
assertEquals(0,split.getOffset(0));
assertEquals(0,split.getOffset(1));
assertTrue(3 >= split.getLocations().length);
org.apache.hadoop.mapred.RecordReader reader=inputFormat.getRecordReader(split,conf,Reporter.NULL);
CombineHiveKey key=reader.createKey();
OrcStruct value=reader.createValue();
for (int i=0; i < 20; i++) {
assertEquals(true,reader.next(key,value));
assertEquals(i,((IntWritable)value.getFieldValue(0)).get());
}
assertEquals(false,reader.next(key,value));
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier (),true,null,null));
List results=splitter.call();
OrcSplit result=results.get(0);
assertEquals(3,result.getStart());
assertEquals(497,result.getLength());
result=results.get(1);
assertEquals(500,result.getStart());
assertEquals(600,result.getLength());
result=results.get(2);
assertEquals(1100,result.getStart());
assertEquals(400,result.getLength());
result=results.get(3);
assertEquals(1500,result.getStart());
assertEquals(300,result.getLength());
result=results.get(4);
assertEquals(1800,result.getStart());
assertEquals(200,result.getLength());
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMAXSPLITSIZE,0);
HiveConf.setLongVar(conf,HiveConf.ConfVars.MAPREDMINSPLITSIZE,0);
context=new OrcInputFormat.Context(conf);
splitter=new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context,fs,AcidUtils.createOriginalObj(null,fs.getFileStatus(new Path("/a/file"))),null,true,new ArrayList(),true,null,null));
results=splitter.call();
for (int i=0; i < stripeSizes.length; ++i) {
assertEquals("checking stripe " + i + " size",stripeSizes[i],results.get(i).getLength());
}
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ",inspector.getTypeName());
InputFormat in=new OrcInputFormat();
FileInputFormat.setInputPaths(conf,testFilePath.toString());
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
conf.set("columns",StringRow.getColumnNamesProperty());
conf.set("columns.types",StringRow.getColumnTypesProperty());
org.apache.hadoop.mapred.RecordReader reader=in.getRecordReader(splits[0],conf,Reporter.NULL);
Object key=reader.createKey();
Writable value=(Writable)reader.createValue();
List fields=inspector.getAllStructFieldRefs();
StringObjectInspector strInspector=(StringObjectInspector)fields.get(0).getFieldObjectInspector();
assertEquals(true,reader.next(key,value));
assertEquals("owen",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(true,reader.next(key,value));
assertEquals("beth",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(true,reader.next(key,value));
assertEquals("laurel",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(true,reader.next(key,value));
assertEquals("hazen",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(true,reader.next(key,value));
assertEquals("colin",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(true,reader.next(key,value));
assertEquals("miles",strInspector.getPrimitiveJavaObject(inspector.getStructFieldData(value,fields.get(0))));
assertEquals(false,reader.next(key,value));
reader.close();
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier (),true,null,null));
OrcSplit result=splitter.createSplit(0,200,null);
assertEquals(0,result.getStart());
assertEquals(200,result.getLength());
assertEquals("mock:/a/file",result.getPath().toString());
String[] locs=result.getLocations();
assertEquals(3,locs.length);
assertEquals("host1-1",locs[0]);
assertEquals("host1-2",locs[1]);
assertEquals("host1-3",locs[2]);
result=splitter.createSplit(500,600,null);
locs=result.getLocations();
assertEquals(3,locs.length);
assertEquals("host2-1",locs[0]);
assertEquals("host0",locs[1]);
assertEquals("host2-3",locs[2]);
result=splitter.createSplit(0,2500,null);
locs=result.getLocations();
assertEquals(1,locs.length);
assertEquals("host0",locs[0]);
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier ();
InputSplit[] splits=inputFormat.getSplits(conf,10);
assertEquals(1,splits.length);
org.apache.hadoop.mapred.RecordReader reader=inputFormat.getRecordReader(splits[0],conf,Reporter.NULL);
NullWritable key=reader.createKey();
VectorizedRowBatch value=reader.createValue();
assertEquals(true,reader.next(key,value));
assertEquals(10,value.count());
LongColumnVector col0=(LongColumnVector)value.cols[0];
for (int i=0; i < 10; i++) {
assertEquals("checking " + i,i,col0.vector[i]);
}
assertEquals(false,reader.next(key,value));
}
Class: org.apache.hadoop.hive.ql.io.orc.TestNewInputOutputFormat APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier map=(Map)converted.get(3);
assertEquals(map.size(),1);
assertEquals(map.get("saving"),new Integer(1));
row=rows.next(null);
converted=(List)converter.convert(row);
assertEquals(2,converted.get(0));
assertEquals(6,converted.get(1));
list=(List)converted.get(2);
assertEquals(list.size(),6);
assertEquals("breakfast",((List)list.get(0)).get(0));
assertEquals(9,((List)list.get(0)).get(1));
map=(Map)converted.get(3);
assertEquals(map.size(),11);
assertEquals(map.get("the"),new Integer(2));
row=rows.next(null);
converted=(List)converter.convert(row);
assertEquals(3,converted.get(0));
assertEquals(5,converted.get(1));
list=(List)converted.get(2);
assertEquals(list.size(),5);
assertEquals("cold",((List)list.get(0)).get(0));
assertEquals(4,((List)list.get(0)).get(1));
map=(Map)converted.get(3);
assertEquals(map.size(),13);
assertEquals(map.get("were"),new Integer(3));
assertFalse(rows.hasNext());
localFs.delete(outputPath,true);
}
Class: org.apache.hadoop.hive.ql.io.orc.TestNewIntegerEncoding APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 100000; i++) {
input.add((long)rand.nextInt());
}
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE).stripeSize(100000).bufferSize(10000).version(OrcFile.Version.V_0_11).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=55555;
rows.seekToRow(idx);
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
input.add(371946367L);
input.add(11963367L);
input.add(68639400007L);
input.add(100233367L);
input.add(6367L);
input.add(10026367L);
input.add(3670000L);
input.add(3602367L);
input.add(4719226367L);
input.add(7196367L);
input.add(444442L);
input.add(210267L);
input.add(21033L);
input.add(160267L);
input.add(400267L);
input.add(23634347L);
input.add(16027L);
input.add(46026367L);
input.add(Long.MAX_VALUE);
input.add(33333L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(60));
}
input.set(128,Long.MAX_VALUE);
input.set(256,Long.MAX_VALUE);
input.set(511,Long.MAX_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 100000; i++) {
input.add((long)rand.nextInt());
}
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
input.add((long)Integer.MAX_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(0,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
input.add(Long.MIN_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
input.add((long)Integer.MIN_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier tslist=Lists.newArrayList();
tslist.add(Timestamp.valueOf("2099-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("1999-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("1995-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2010-03-02 00:00:00"));
tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("1996-08-02 00:00:00"));
tslist.add(Timestamp.valueOf("1998-11-02 00:00:00"));
tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
tslist.add(Timestamp.valueOf("1993-08-02 00:00:00"));
tslist.add(Timestamp.valueOf("2008-01-02 00:00:00"));
tslist.add(Timestamp.valueOf("2007-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2008-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("1994-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2001-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2011-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
tslist.add(Timestamp.valueOf("1974-01-01 00:00:00"));
for ( Timestamp ts : tslist) {
writer.addRow(new TSRow(ts));
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(tslist.get(idx++).getNanos(),((TimestampWritable)((OrcStruct)row).getFieldValue(0)).getNanos());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(1,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
for (int i=0; i < 25; i++) {
input.add(371292224226367L);
input.add(119622332222267L);
input.add(686329400222007L);
input.add(100233333222367L);
input.add(636272333322222L);
input.add(10202633223267L);
input.add(36700222022230L);
input.add(36023226224227L);
input.add(47192226364427L);
input.add(71963622222447L);
input.add(22244444222222L);
input.add(21220263327442L);
input.add(21032233332232L);
input.add(16026322232227L);
input.add(40022262272212L);
input.add(23634342227222L);
input.add(16022222222227L);
input.add(46026362222227L);
input.add(46026362222227L);
input.add(33322222222323L);
}
input.add(Long.MAX_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
input.add(Long.MAX_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 100000; i++) {
input.add(rand.nextLong());
}
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(256,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(255,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(60));
}
input.set(511,Long.MAX_VALUE);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(510,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE).version(OrcFile.Version.V_0_11).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList();
Random rand=new Random();
for (int i=0; i < 5120; i++) {
input.add((long)rand.nextInt(100));
}
input.set(511,20000L);
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000).encodingStrategy(encodingStrategy));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
Class: org.apache.hadoop.hive.ql.io.orc.TestOrcFile APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("foo",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("bar",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3,4,5),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertNull(st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertNull(bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier tslist=Lists.newArrayList();
tslist.add(Timestamp.valueOf("2037-01-01 00:00:00.000999"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.000000222"));
tslist.add(Timestamp.valueOf("1999-01-01 00:00:00.999999999"));
tslist.add(Timestamp.valueOf("1995-01-01 00:00:00.688888888"));
tslist.add(Timestamp.valueOf("2002-01-01 00:00:00.1"));
tslist.add(Timestamp.valueOf("2010-03-02 00:00:00.000009001"));
tslist.add(Timestamp.valueOf("2005-01-01 00:00:00.000002229"));
tslist.add(Timestamp.valueOf("2006-01-01 00:00:00.900203003"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.800000007"));
tslist.add(Timestamp.valueOf("1996-08-02 00:00:00.723100809"));
tslist.add(Timestamp.valueOf("1998-11-02 00:00:00.857340643"));
tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
for ( Timestamp ts : tslist) {
writer.addRow(ts);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows(null);
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(tslist.get(idx++).getNanos(),((TimestampWritable)row).getNanos());
}
assertEquals(0,writer.getSchema().getMaximumId());
boolean[] expected=new boolean[]{false};
boolean[] included=OrcUtils.includeColumns("",writer.getSchema());
assertEquals(true,Arrays.equals(expected,included));
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier expectedList=expected.middle.list;
List actualList=(List)((OrcStruct)row.getFieldValue(9)).getFieldValue(0);
compareList(expectedList,actualList);
compareList(expected.list,(List)row.getFieldValue(10));
}
rows.close();
Iterator stripeIterator=reader.getStripes().iterator();
long offsetOfStripe2=0;
long offsetOfStripe4=0;
long lastRowOfStripe2=0;
for (int i=0; i < 5; ++i) {
StripeInformation stripe=stripeIterator.next();
if (i < 2) {
lastRowOfStripe2+=stripe.getNumberOfRows();
}
else if (i == 2) {
offsetOfStripe2=stripe.getOffset();
lastRowOfStripe2+=stripe.getNumberOfRows() - 1;
}
else if (i == 4) {
offsetOfStripe4=stripe.getOffset();
}
}
boolean[] columns=new boolean[reader.getStatistics().length];
columns[5]=true;
columns[9]=true;
rows=reader.rowsOptions(new Reader.Options().range(offsetOfStripe2,offsetOfStripe4 - offsetOfStripe2).include(columns));
rows.seekToRow(lastRowOfStripe2);
for (int i=0; i < 2; ++i) {
row=(OrcStruct)rows.next(row);
BigRow expected=createRandomRow(intValues,doubleValues,stringValues,byteValues,words,(int)(lastRowOfStripe2 + i));
assertEquals(expected.long1.longValue(),((LongWritable)row.getFieldValue(4)).get());
assertEquals(expected.string1,row.getFieldValue(8));
}
rows.close();
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
Assert.assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier expectedList=expected.middle.list;
List actualList=(List)((OrcStruct)row.getFieldValue(9)).getFieldValue(0);
compareList(expectedList,actualList);
compareList(expected.list,(List)row.getFieldValue(10));
}
rows.close();
Iterator stripeIterator=reader.getStripes().iterator();
long offsetOfStripe2=0;
long offsetOfStripe4=0;
long lastRowOfStripe2=0;
for (int i=0; i < 5; ++i) {
StripeInformation stripe=stripeIterator.next();
if (i < 2) {
lastRowOfStripe2+=stripe.getNumberOfRows();
}
else if (i == 2) {
offsetOfStripe2=stripe.getOffset();
lastRowOfStripe2+=stripe.getNumberOfRows() - 1;
}
else if (i == 4) {
offsetOfStripe4=stripe.getOffset();
}
}
boolean[] columns=new boolean[reader.getStatistics().length];
columns[5]=true;
columns[9]=true;
rows=reader.rowsOptions(new Reader.Options().range(offsetOfStripe2,offsetOfStripe4 - offsetOfStripe2).include(columns));
rows.seekToRow(lastRowOfStripe2);
for (int i=0; i < 2; ++i) {
row=(OrcStruct)rows.next(row);
BigRow expected=createRandomRow(intValues,doubleValues,stringValues,byteValues,words,(int)(lastRowOfStripe2 + i));
assertEquals(expected.long1.longValue(),((LongWritable)row.getFieldValue(4)).get());
assertEquals(expected.string1,row.getFieldValue(8));
}
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>,list:array>,"+ "map:map>>",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BooleanObjectInspector bo=(BooleanObjectInspector)readerInspector.getStructFieldRef("boolean1").getFieldObjectInspector();
ByteObjectInspector by=(ByteObjectInspector)readerInspector.getStructFieldRef("byte1").getFieldObjectInspector();
ShortObjectInspector sh=(ShortObjectInspector)readerInspector.getStructFieldRef("short1").getFieldObjectInspector();
IntObjectInspector in=(IntObjectInspector)readerInspector.getStructFieldRef("int1").getFieldObjectInspector();
LongObjectInspector lo=(LongObjectInspector)readerInspector.getStructFieldRef("long1").getFieldObjectInspector();
FloatObjectInspector fl=(FloatObjectInspector)readerInspector.getStructFieldRef("float1").getFieldObjectInspector();
DoubleObjectInspector dbl=(DoubleObjectInspector)readerInspector.getStructFieldRef("double1").getFieldObjectInspector();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
StructObjectInspector mid=(StructObjectInspector)readerInspector.getStructFieldRef("middle").getFieldObjectInspector();
List midFields=mid.getAllStructFieldRefs();
ListObjectInspector midli=(ListObjectInspector)midFields.get(0).getFieldObjectInspector();
StructObjectInspector inner=(StructObjectInspector)midli.getListElementObjectInspector();
List inFields=inner.getAllStructFieldRefs();
ListObjectInspector li=(ListObjectInspector)readerInspector.getStructFieldRef("list").getFieldObjectInspector();
MapObjectInspector ma=(MapObjectInspector)readerInspector.getStructFieldRef("map").getFieldObjectInspector();
StringObjectInspector mk=(StringObjectInspector)ma.getMapKeyObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(false,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(1,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(1024,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(1.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-15.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
List midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
List list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(2,list.size());
assertEquals(3,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("good",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(4,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("bad",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
Map map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(0,map.size());
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(true,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(100,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(2048,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(2.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-5.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("bye",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(3,list.size());
assertEquals(100000000,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("cat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(-100000,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("in",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
assertEquals(1234,in.get(inner.getStructFieldData(list.get(2),inFields.get(0))));
assertEquals("hat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(2),inFields.get(1))));
map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(2,map.size());
boolean[] found=new boolean[2];
for ( Object key : map.keySet()) {
String str=mk.getPrimitiveJavaObject(key);
if (str.equals("chani")) {
assertEquals(false,found[0]);
assertEquals(5,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[0]=true;
}
else if (str.equals("mauddib")) {
assertEquals(false,found[1]);
assertEquals(1,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[1]=true;
}
else {
throw new IllegalArgumentException("Unknown key " + str);
}
}
assertEquals(true,found[0]);
assertEquals(true,found[1]);
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier stats=reader.getStripeStatistics();
int numStripes=stats.size();
assertEquals(3,numStripes);
StripeStatistics ss1=stats.get(0);
StripeStatistics ss2=stats.get(1);
StripeStatistics ss3=stats.get(2);
assertEquals(5000,ss1.getColumnStatistics()[0].getNumberOfValues());
assertEquals(5000,ss2.getColumnStatistics()[0].getNumberOfValues());
assertEquals(1000,ss3.getColumnStatistics()[0].getNumberOfValues());
assertEquals(5000,(ss1.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(5000,(ss2.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(1000,(ss3.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(1,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMinimum());
assertEquals(2,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMinimum());
assertEquals(3,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMinimum());
assertEquals(1,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMaximum());
assertEquals(2,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMaximum());
assertEquals(3,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMaximum());
assertEquals(5000,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getSum());
assertEquals(10000,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getSum());
assertEquals(3000,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getSum());
assertEquals(5000,(ss1.getColumnStatistics()[2]).getNumberOfValues());
assertEquals(5000,(ss2.getColumnStatistics()[2]).getNumberOfValues());
assertEquals(1000,(ss3.getColumnStatistics()[2]).getNumberOfValues());
assertEquals("one",((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMinimum());
assertEquals("two",((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMinimum());
assertEquals("three",((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMinimum());
assertEquals("one",((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMaximum());
assertEquals("two",((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMaximum());
assertEquals("three",((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMaximum());
assertEquals(15000,((StringColumnStatistics)ss1.getColumnStatistics()[2]).getSum());
assertEquals(15000,((StringColumnStatistics)ss2.getColumnStatistics()[2]).getSum());
assertEquals(5000,((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum());
RecordReaderImpl recordReader=(RecordReaderImpl)reader.rows();
OrcProto.RowIndex[] index=recordReader.readRowIndex(0,null,null).getRowGroupIndex();
assertEquals(3,index.length);
List items=index[1].getEntryList();
assertEquals(1,items.size());
assertEquals(3,items.get(0).getPositionsCount());
assertEquals(0,items.get(0).getPositions(0));
assertEquals(0,items.get(0).getPositions(1));
assertEquals(0,items.get(0).getPositions(2));
assertEquals(1,items.get(0).getStatistics().getIntStatistics().getMinimum());
index=recordReader.readRowIndex(1,null,null).getRowGroupIndex();
assertEquals(3,index.length);
items=index[1].getEntryList();
assertEquals(2,items.get(0).getStatistics().getIntStatistics().getMaximum());
}
APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier types=new ArrayList();
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRUCT).addFieldNames("time").addFieldNames("union").addFieldNames("decimal").addSubtypes(1).addSubtypes(2).addSubtypes(5).build());
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.TIMESTAMP).build());
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.UNION).addSubtypes(3).addSubtypes(4).build());
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.INT).build());
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRING).build());
types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.DECIMAL).build());
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector=OrcStruct.createObjectInspector(0,types);
}
HiveDecimal maxValue=HiveDecimal.create("10000000000000000000");
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(1000).compress(CompressionKind.NONE).batchSize(1000).bufferSize(100).blockPadding(false));
OrcStruct row=new OrcStruct(3);
OrcUnion union=new OrcUnion();
row.setFieldValue(1,union);
row.setFieldValue(0,new TimestampWritable(Timestamp.valueOf("2000-03-12 15:00:00")));
HiveDecimal value=HiveDecimal.create("12345678.6547456");
row.setFieldValue(2,new HiveDecimalWritable(value));
union.set((byte)0,new IntWritable(42));
writer.addRow(row);
row.setFieldValue(0,new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")));
union.set((byte)1,new Text("hello"));
value=HiveDecimal.create("-5643.234");
row.setFieldValue(2,new HiveDecimalWritable(value));
writer.addRow(row);
row.setFieldValue(0,null);
row.setFieldValue(1,null);
row.setFieldValue(2,null);
writer.addRow(row);
row.setFieldValue(1,union);
union.set((byte)0,null);
writer.addRow(row);
union.set((byte)1,null);
writer.addRow(row);
union.set((byte)0,new IntWritable(200000));
row.setFieldValue(0,new TimestampWritable(Timestamp.valueOf("1970-01-01 00:00:00")));
value=HiveDecimal.create("10000000000000000000");
row.setFieldValue(2,new HiveDecimalWritable(value));
writer.addRow(row);
Random rand=new Random(42);
for (int i=1970; i < 2038; ++i) {
row.setFieldValue(0,new TimestampWritable(Timestamp.valueOf(i + "-05-05 12:34:56." + i)));
if ((i & 1) == 0) {
union.set((byte)0,new IntWritable(i * i));
}
else {
union.set((byte)1,new Text(Integer.toString(i * i)));
}
value=HiveDecimal.create(new BigInteger(64,rand),rand.nextInt(18));
row.setFieldValue(2,new HiveDecimalWritable(value));
if (maxValue.compareTo(value) < 0) {
maxValue=value;
}
writer.addRow(row);
}
row.setFieldValue(0,null);
union.set((byte)0,new IntWritable(1732050807));
row.setFieldValue(2,null);
for (int i=0; i < 5000; ++i) {
writer.addRow(row);
}
union.set((byte)0,new IntWritable(0));
writer.addRow(row);
union.set((byte)0,new IntWritable(10));
writer.addRow(row);
union.set((byte)0,new IntWritable(138));
writer.addRow(row);
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
TypeDescription schema=writer.getSchema();
assertEquals(5,schema.getMaximumId());
boolean[] expected=new boolean[]{false,false,false,false,false,false};
boolean[] included=OrcUtils.includeColumns("",schema);
assertEquals(true,Arrays.equals(expected,included));
expected=new boolean[]{false,true,false,false,false,true};
included=OrcUtils.includeColumns("time,decimal",schema);
assertEquals(true,Arrays.equals(expected,included));
expected=new boolean[]{false,false,true,true,true,false};
included=OrcUtils.includeColumns("union",schema);
assertEquals(true,Arrays.equals(expected,included));
assertEquals(false,reader.getMetadataKeys().iterator().hasNext());
assertEquals(5077,reader.getNumberOfRows());
DecimalColumnStatistics stats=(DecimalColumnStatistics)reader.getStatistics()[5];
assertEquals(71,stats.getNumberOfValues());
assertEquals(HiveDecimal.create("-5643.234"),stats.getMinimum());
assertEquals(maxValue,stats.getMaximum());
int stripeCount=0;
int rowCount=0;
long currentOffset=-1;
for ( StripeInformation stripe : reader.getStripes()) {
stripeCount+=1;
rowCount+=stripe.getNumberOfRows();
if (currentOffset < 0) {
currentOffset=stripe.getOffset() + stripe.getLength();
}
else {
assertEquals(currentOffset,stripe.getOffset());
currentOffset+=stripe.getLength();
}
}
assertEquals(reader.getNumberOfRows(),rowCount);
assertEquals(2,stripeCount);
assertEquals(reader.getContentLength(),currentOffset);
RecordReader rows=reader.rows();
assertEquals(0,rows.getRowNumber());
assertEquals(0.0,rows.getProgress(),0.000001);
assertEquals(true,rows.hasNext());
row=(OrcStruct)rows.next(null);
assertEquals(1,rows.getRowNumber());
inspector=reader.getObjectInspector();
assertEquals("struct,decimal:decimal(38,18)>",inspector.getTypeName());
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-12 15:00:00")),row.getFieldValue(0));
union=(OrcUnion)row.getFieldValue(1);
assertEquals(0,union.getTag());
assertEquals(new IntWritable(42),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("12345678.6547456")),row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(2,rows.getRowNumber());
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(new Text("hello"),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")),row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
assertEquals(null,row.getFieldValue(1));
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
union=(OrcUnion)row.getFieldValue(1);
assertEquals(0,union.getTag());
assertEquals(null,union.getObject());
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(null,union.getObject());
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf("1970-01-01 00:00:00")),row.getFieldValue(0));
assertEquals(new IntWritable(200000),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("10000000000000000000")),row.getFieldValue(2));
rand=new Random(42);
for (int i=1970; i < 2038; ++i) {
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf(i + "-05-05 12:34:56." + i)),row.getFieldValue(0));
if ((i & 1) == 0) {
assertEquals(0,union.getTag());
assertEquals(new IntWritable(i * i),union.getObject());
}
else {
assertEquals(1,union.getTag());
assertEquals(new Text(Integer.toString(i * i)),union.getObject());
}
assertEquals(new HiveDecimalWritable(HiveDecimal.create(new BigInteger(64,rand),rand.nextInt(18))),row.getFieldValue(2));
}
for (int i=0; i < 5000; ++i) {
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(1732050807),union.getObject());
}
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(0),union.getObject());
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(10),union.getObject());
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(138),union.getObject());
assertEquals(false,rows.hasNext());
assertEquals(1.0,rows.getProgress(),0.00001);
assertEquals(reader.getNumberOfRows(),rows.getRowNumber());
rows.seekToRow(1);
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(new Text("hello"),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")),row.getFieldValue(2));
rows.close();
}
APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>,list:array>,"+ "map:map>,ts:timestamp,"+ "decimal1:decimal(38,18)>",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BooleanObjectInspector bo=(BooleanObjectInspector)readerInspector.getStructFieldRef("boolean1").getFieldObjectInspector();
ByteObjectInspector by=(ByteObjectInspector)readerInspector.getStructFieldRef("byte1").getFieldObjectInspector();
ShortObjectInspector sh=(ShortObjectInspector)readerInspector.getStructFieldRef("short1").getFieldObjectInspector();
IntObjectInspector in=(IntObjectInspector)readerInspector.getStructFieldRef("int1").getFieldObjectInspector();
LongObjectInspector lo=(LongObjectInspector)readerInspector.getStructFieldRef("long1").getFieldObjectInspector();
FloatObjectInspector fl=(FloatObjectInspector)readerInspector.getStructFieldRef("float1").getFieldObjectInspector();
DoubleObjectInspector dbl=(DoubleObjectInspector)readerInspector.getStructFieldRef("double1").getFieldObjectInspector();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
StructObjectInspector mid=(StructObjectInspector)readerInspector.getStructFieldRef("middle").getFieldObjectInspector();
List midFields=mid.getAllStructFieldRefs();
ListObjectInspector midli=(ListObjectInspector)midFields.get(0).getFieldObjectInspector();
StructObjectInspector inner=(StructObjectInspector)midli.getListElementObjectInspector();
List inFields=inner.getAllStructFieldRefs();
ListObjectInspector li=(ListObjectInspector)readerInspector.getStructFieldRef("list").getFieldObjectInspector();
MapObjectInspector ma=(MapObjectInspector)readerInspector.getStructFieldRef("map").getFieldObjectInspector();
TimestampObjectInspector tso=(TimestampObjectInspector)readerInspector.getStructFieldRef("ts").getFieldObjectInspector();
HiveDecimalObjectInspector dco=(HiveDecimalObjectInspector)readerInspector.getStructFieldRef("decimal1").getFieldObjectInspector();
StringObjectInspector mk=(StringObjectInspector)ma.getMapKeyObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(false,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(1,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(1024,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(1.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-15.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
List midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
List list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(2,list.size());
assertEquals(3,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("good",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(4,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("bad",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
Map map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(0,map.size());
assertEquals(Timestamp.valueOf("2000-03-12 15:00:00"),tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(12))));
assertEquals(HiveDecimal.create("12345678.6547456"),dco.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(13))));
assertEquals(true,rows.hasNext());
rows.seekToRow(7499);
row=rows.next(null);
assertEquals(true,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(100,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(2048,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(2.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-5.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("bye",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(3,list.size());
assertEquals(100000000,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("cat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(-100000,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("in",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
assertEquals(1234,in.get(inner.getStructFieldData(list.get(2),inFields.get(0))));
assertEquals("hat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(2),inFields.get(1))));
map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(2,map.size());
boolean[] found=new boolean[2];
for ( Object key : map.keySet()) {
String str=mk.getPrimitiveJavaObject(key);
if (str.equals("chani")) {
assertEquals(false,found[0]);
assertEquals(5,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[0]=true;
}
else if (str.equals("mauddib")) {
assertEquals(false,found[1]);
assertEquals(1,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[1]=true;
}
else {
throw new IllegalArgumentException("Unknown key " + str);
}
}
assertEquals(true,found[0]);
assertEquals(true,found[1]);
assertEquals(Timestamp.valueOf("2000-03-12 15:00:01"),tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(12))));
assertEquals(HiveDecimal.create("12345678.6547457"),dco.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(13))));
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier types=reader.getTypes();
assertEquals(3,types.size());
assertEquals(OrcProto.Type.Kind.STRUCT,types.get(0).getKind());
assertEquals(2,types.get(0).getSubtypesCount());
assertEquals(1,types.get(0).getSubtypes(0));
assertEquals(2,types.get(0).getSubtypes(1));
assertEquals(OrcProto.Type.Kind.INT,types.get(1).getKind());
assertEquals(0,types.get(1).getSubtypesCount());
assertEquals(OrcProto.Type.Kind.STRING,types.get(2).getKind());
assertEquals(0,types.get(2).getSubtypesCount());
RecordReader rows1=reader.rows(new boolean[]{true,true,false});
RecordReader rows2=reader.rows(new boolean[]{true,false,true});
r1=new Random(1);
r2=new Random(2);
OrcStruct row1=null;
OrcStruct row2=null;
for (int i=0; i < 21000; ++i) {
assertEquals(true,rows1.hasNext());
assertEquals(true,rows2.hasNext());
row1=(OrcStruct)rows1.next(row1);
row2=(OrcStruct)rows2.next(row2);
assertEquals(r1.nextInt(),((IntWritable)row1.getFieldValue(0)).get());
assertEquals(Long.toHexString(r2.nextLong()),row2.getFieldValue(1).toString());
}
assertEquals(false,rows1.hasNext());
assertEquals(false,rows2.hasNext());
rows1.close();
rows2.close();
}
Class: org.apache.hadoop.hive.ql.io.orc.TestOrcFileStripeMergeRecordReader EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcNullOptimization APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>",readerInspector.getTypeName());
RecordReader rows=reader.rows();
List expected=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
expected.add(false);
}
expected.set(expected.size() - 1,true);
List got=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
OrcProto.StripeFooter sf=((RecordReaderImpl)rows).readStripeFooter(sinfo);
got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString()) != -1);
}
assertEquals(expected,got);
OrcStruct row=(OrcStruct)rows.next(null);
assertNotNull(row);
assertEquals(new IntWritable(3),row.getFieldValue(0));
assertEquals("a",row.getFieldValue(1).toString());
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
row=(OrcStruct)rows.next(row);
assertNotNull(row);
assertNull(row.getFieldValue(0));
assertEquals("b",row.getFieldValue(1).toString());
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
row=(OrcStruct)rows.next(row);
assertNotNull(row);
assertNull(row.getFieldValue(1));
assertEquals(new IntWritable(3),row.getFieldValue(0));
assertEquals(new BooleanWritable(false),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>",readerInspector.getTypeName());
RecordReader rows=reader.rows();
List expected=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
expected.add(false);
}
List got=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
OrcProto.StripeFooter sf=((RecordReaderImpl)rows).readStripeFooter(sinfo);
got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString()) != -1);
}
assertEquals(expected,got);
rows.seekToRow(19998);
OrcStruct row=(OrcStruct)rows.next(null);
assertNotNull(row);
assertNotNull(row.getFieldValue(1));
assertEquals(new IntWritable(0),row.getFieldValue(0));
assertEquals("a",row.getFieldValue(1).toString());
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
row=(OrcStruct)rows.next(row);
assertNotNull(row);
assertNotNull(row.getFieldValue(0));
assertNotNull(row.getFieldValue(1));
assertEquals("b",row.getFieldValue(1).toString());
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>",readerInspector.getTypeName());
RecordReader rows=reader.rows();
List expected=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
expected.add(false);
}
expected.set(0,true);
expected.set(expected.size() - 1,true);
List got=Lists.newArrayList();
for ( StripeInformation sinfo : reader.getStripes()) {
OrcProto.StripeFooter sf=((RecordReaderImpl)rows).readStripeFooter(sinfo);
got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString()) != -1);
}
assertEquals(expected,got);
OrcStruct row=(OrcStruct)rows.next(null);
assertNotNull(row);
assertNull(row.getFieldValue(0));
assertNull(row.getFieldValue(1));
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
rows.seekToRow(19998);
row=(OrcStruct)rows.next(null);
assertNotNull(row);
assertNotNull(row.getFieldValue(1));
assertEquals(new IntWritable(0),row.getFieldValue(0));
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
row=(OrcStruct)rows.next(row);
assertNotNull(row);
assertNull(row.getFieldValue(0));
assertNull(row.getFieldValue(1));
assertEquals(new BooleanWritable(true),row.getFieldValue(2));
assertEquals(new IntWritable(100),((OrcStruct)((ArrayList)row.getFieldValue(3)).get(0)).getFieldValue(0));
rows.close();
}
Class: org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier rr;
rr=inf.getRecordReader(splits[0],job,Reporter.NULL);
values=new String[]{"a","b","c","d","e","f","g","h","i","j"};
OrcStruct row=rr.createValue();
for (int i=0; i < values.length; ++i) {
System.out.println("Checking " + i);
assertEquals(true,rr.next(NullWritable.get(),row));
assertEquals(values[i],row.getFieldValue(0).toString());
}
assertEquals(false,rr.next(NullWritable.get(),row));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier types=new ArrayList();
OrcProto.Type.Builder typeBuilder=OrcProto.Type.newBuilder();
typeBuilder.setKind(OrcProto.Type.Kind.STRUCT).addSubtypes(1).addSubtypes(2).addSubtypes(3).addSubtypes(4).addSubtypes(5).addSubtypes(6);
typeBuilder.addAllFieldNames(Lists.newArrayList("operation","originalTransaction","bucket","rowId","currentTransaction","row"));
types.add(typeBuilder.build());
types.add(null);
types.add(null);
types.add(null);
types.add(null);
types.add(null);
typeBuilder.clearSubtypes();
typeBuilder.addSubtypes(7);
typeBuilder.addAllFieldNames(Lists.newArrayList("col1"));
types.add(typeBuilder.build());
typeBuilder.clear();
typeBuilder.setKind(OrcProto.Type.Kind.STRING);
types.add(typeBuilder.build());
Mockito.when(reader.getTypes()).thenReturn(types);
Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class))).thenReturn(recordReader);
OrcStruct row1=new OrcStruct(OrcRecordUpdater.FIELDS);
setRow(row1,OrcRecordUpdater.INSERT_OPERATION,10,20,20,100,"first");
OrcStruct row2=new OrcStruct(OrcRecordUpdater.FIELDS);
setRow(row2,OrcRecordUpdater.INSERT_OPERATION,10,20,30,110,"second");
OrcStruct row3=new OrcStruct(OrcRecordUpdater.FIELDS);
setRow(row3,OrcRecordUpdater.INSERT_OPERATION,10,20,40,120,"third");
OrcStruct row4=new OrcStruct(OrcRecordUpdater.FIELDS);
setRow(row4,OrcRecordUpdater.INSERT_OPERATION,40,50,60,130,"fourth");
OrcStruct row5=new OrcStruct(OrcRecordUpdater.FIELDS);
setRow(row5,OrcRecordUpdater.INSERT_OPERATION,40,50,61,140,"fifth");
Mockito.when(recordReader.hasNext()).thenReturn(true,true,true,true,true,false);
Mockito.when(recordReader.getProgress()).thenReturn(1.0f);
Mockito.when(recordReader.next(null)).thenReturn(row1,row4);
Mockito.when(recordReader.next(row1)).thenReturn(row2);
Mockito.when(recordReader.next(row2)).thenReturn(row3);
Mockito.when(recordReader.next(row3)).thenReturn(row5);
Mockito.when(reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)).thenReturn(ByteBuffer.wrap("10,20,30;40,50,60;40,50,61".getBytes("UTF-8")));
Mockito.when(reader.getStripes()).thenReturn(createStripes(2,2,1));
OrcRawRecordMerger merger=new OrcRawRecordMerger(conf,false,reader,false,10,createMaximalTxnList(),new Reader.Options().range(1000,1000),null);
RecordReader rr=merger.getCurrentReader().recordReader;
assertEquals(0,merger.getOtherReaders().size());
assertEquals(new RecordIdentifier(10,20,30),merger.getMinKey());
assertEquals(new RecordIdentifier(40,50,60),merger.getMaxKey());
RecordIdentifier id=merger.createKey();
OrcStruct event=merger.createValue();
assertEquals(true,merger.next(id,event));
assertEquals(10,id.getTransactionId());
assertEquals(20,id.getBucketId());
assertEquals(40,id.getRowId());
assertEquals("third",getValue(event));
assertEquals(true,merger.next(id,event));
assertEquals(40,id.getTransactionId());
assertEquals(50,id.getBucketId());
assertEquals(60,id.getRowId());
assertEquals("fourth",getValue(event));
assertEquals(false,merger.next(id,event));
assertEquals(1.0,merger.getProgress(),0.01);
merger.close();
Mockito.verify(rr).close();
Mockito.verify(rr).getProgress();
StructObjectInspector eventObjectInspector=(StructObjectInspector)merger.getObjectInspector();
List fields=eventObjectInspector.getAllStructFieldRefs();
assertEquals(OrcRecordUpdater.FIELDS,fields.size());
assertEquals("operation",fields.get(OrcRecordUpdater.OPERATION).getFieldName());
assertEquals("currentTransaction",fields.get(OrcRecordUpdater.CURRENT_TRANSACTION).getFieldName());
assertEquals("originalTransaction",fields.get(OrcRecordUpdater.ORIGINAL_TRANSACTION).getFieldName());
assertEquals("bucket",fields.get(OrcRecordUpdater.BUCKET).getFieldName());
assertEquals("rowId",fields.get(OrcRecordUpdater.ROW_ID).getFieldName());
StructObjectInspector rowObjectInspector=(StructObjectInspector)fields.get(OrcRecordUpdater.ROW).getFieldObjectInspector();
assertEquals("col1",rowObjectInspector.getAllStructFieldRefs().get(0).getFieldName());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier rr;
for (int i=0; i < 4; ++i) {
System.out.println("starting split " + i + " = "+ splits[i]);
rr=inf.getRecordReader(splits[i],job,Reporter.NULL);
NullWritable key=rr.createKey();
OrcStruct value=rr.createValue();
for (int j=0; j < 2; ++j) {
System.out.println("i = " + i + ", j = "+ j);
assertEquals(true,rr.next(key,value));
System.out.println("record = " + value);
assertEquals(i + "." + j,value.getFieldValue(2).toString());
}
assertEquals(false,rr.next(key,value));
}
rr=inf.getRecordReader(splits[4],job,Reporter.NULL);
assertEquals(false,rr.next(rr.createKey(),rr.createValue()));
}
InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier rr;
for (int i=0; i < 4; ++i) {
System.out.println("starting split " + i);
rr=inf.getRecordReader(splits[i],job,Reporter.NULL);
NullWritable key=rr.createKey();
OrcStruct value=rr.createValue();
for (int j=0; j < 2; ++j) {
System.out.println("i = " + i + ", j = "+ j);
assertEquals(true,rr.next(key,value));
System.out.println("record = " + value);
assertEquals(i + "." + j,value.getFieldValue(2).toString());
}
assertEquals(false,rr.next(key,value));
}
rr=inf.getRecordReader(splits[4],job,Reporter.NULL);
assertEquals(false,rr.next(rr.createKey(),rr.createValue()));
}
InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcRecordUpdater APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcSerDeStats APIUtilityVerifier InternalCallVerifier EqualityVerifier test=new HashMap();
for (int i=0; i < 10; i++) {
test.put("hi" + i,2.0);
}
writer.addRow(new MapStruct(test));
}
writer.close();
assertEquals(1000,writer.getNumberOfRows());
assertEquals(950000,writer.getRawDataSize());
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(1000,reader.getNumberOfRows());
assertEquals(950000,reader.getRawDataSize());
assertEquals(950000,reader.getRawDataSizeOfColumns(Lists.newArrayList("map1")));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier ExceptionVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("foo",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("bar",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3,4,5),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertNull(st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertNull(bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier test=new ArrayList();
for (int i=0; i < 1000; i++) {
test.add("hi");
}
writer.addRow(new ListStruct(test));
}
writer.close();
assertEquals(5000,writer.getNumberOfRows());
assertEquals(430000000,writer.getRawDataSize());
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(5000,reader.getNumberOfRows());
assertEquals(430000000,reader.getRawDataSize());
assertEquals(430000000,reader.getRawDataSizeOfColumns(Lists.newArrayList("list1")));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcSplitElimination InternalCallVerifier EqualityVerifier childExpr=Lists.newArrayList();
ExprNodeColumnDesc col=new ExprNodeColumnDesc(Long.class,"userid","T",false);
ExprNodeConstantDesc con=new ExprNodeConstantDesc(100);
childExpr.add(col);
childExpr.add(con);
ExprNodeGenericFuncDesc en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
String sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(0);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(0,splits.length);
con=new ExprNodeConstantDesc(2);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
con=new ExprNodeConstantDesc(5);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(13);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(29);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(70);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
}
InternalCallVerifier EqualityVerifier childExpr=Lists.newArrayList();
ExprNodeColumnDesc col=new ExprNodeColumnDesc(Long.class,"userid","T",false);
ExprNodeConstantDesc con=new ExprNodeConstantDesc(100);
childExpr.add(col);
childExpr.add(con);
ExprNodeGenericFuncDesc en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
String sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(5,splits.length);
con=new ExprNodeConstantDesc(1);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(0,splits.length);
con=new ExprNodeConstantDesc(2);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
con=new ExprNodeConstantDesc(5);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(13);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(3,splits.length);
con=new ExprNodeConstantDesc(29);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(4,splits.length);
con=new ExprNodeConstantDesc(70);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
sargStr=SerializationUtilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(5,splits.length);
}
InternalCallVerifier EqualityVerifier childExpr=Lists.newArrayList();
ExprNodeColumnDesc col=new ExprNodeColumnDesc(Long.class,"userid","T",false);
ExprNodeConstantDesc con=new ExprNodeConstantDesc(100);
childExpr.add(col);
childExpr.add(con);
ExprNodeGenericFuncDesc en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
GenericUDF udf1=new GenericUDFOPEqualOrLessThan();
List childExpr1=Lists.newArrayList();
ExprNodeColumnDesc col1=new ExprNodeColumnDesc(Double.class,"subtype","T",false);
ExprNodeConstantDesc con1=new ExprNodeConstantDesc(1000.0);
childExpr1.add(col1);
childExpr1.add(con1);
ExprNodeGenericFuncDesc en1=new ExprNodeGenericFuncDesc(inspector,udf1,childExpr1);
GenericUDF udf2=new GenericUDFOPAnd();
List childExpr2=Lists.newArrayList();
childExpr2.add(en);
childExpr2.add(en1);
ExprNodeGenericFuncDesc en2=new ExprNodeGenericFuncDesc(inspector,udf2,childExpr2);
String sargStr=SerializationUtilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized",sargStr);
InputSplit[] splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
con=new ExprNodeConstantDesc(2);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
con1=new ExprNodeConstantDesc(0.0);
childExpr1.set(1,con1);
en1=new ExprNodeGenericFuncDesc(inspector,udf1,childExpr1);
childExpr2.set(0,en);
childExpr2.set(1,en1);
en2=new ExprNodeGenericFuncDesc(inspector,udf2,childExpr2);
sargStr=SerializationUtilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(0,splits.length);
con=new ExprNodeConstantDesc(2);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
con1=new ExprNodeConstantDesc(1.0);
childExpr1.set(1,con1);
en1=new ExprNodeGenericFuncDesc(inspector,udf1,childExpr1);
childExpr2.set(0,en);
childExpr2.set(1,en1);
en2=new ExprNodeGenericFuncDesc(inspector,udf2,childExpr2);
sargStr=SerializationUtilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
udf=new GenericUDFOPEqual();
con=new ExprNodeConstantDesc(13);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
con1=new ExprNodeConstantDesc(80.0);
childExpr1.set(1,con1);
en1=new ExprNodeGenericFuncDesc(inspector,udf1,childExpr1);
childExpr2.set(0,en);
childExpr2.set(1,en1);
en2=new ExprNodeGenericFuncDesc(inspector,udf2,childExpr2);
sargStr=SerializationUtilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(2,splits.length);
udf=new GenericUDFOPEqual();
con=new ExprNodeConstantDesc(13);
childExpr.set(1,con);
en=new ExprNodeGenericFuncDesc(inspector,udf,childExpr);
udf1=new GenericUDFOPEqual();
con1=new ExprNodeConstantDesc(80.0);
childExpr1.set(1,con1);
en1=new ExprNodeGenericFuncDesc(inspector,udf1,childExpr1);
childExpr2.set(0,en);
childExpr2.set(1,en1);
en2=new ExprNodeGenericFuncDesc(inspector,udf2,childExpr2);
sargStr=SerializationUtilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized",sargStr);
splits=in.getSplits(conf,1);
assertEquals(1,splits.length);
}
Class: org.apache.hadoop.hive.ql.io.orc.TestOrcStruct InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier ,c11:map,c12:uniontype"+ ",c13:array>");
StructObjectInspector inspector=(StructObjectInspector)OrcStruct.createObjectInspector(typeInfo);
assertEquals("struct,c11:map,c12:uniontype,c13:array>",inspector.getTypeName());
assertEquals(null,inspector.getAllStructFieldRefs().get(0).getFieldComment());
assertEquals(null,inspector.getStructFieldRef("UNKNOWN"));
OrcStruct s1=new OrcStruct(13);
for (int i=0; i < 13; ++i) {
s1.setFieldValue(i,i);
}
List list=new ArrayList();
list.addAll(Arrays.asList(0,1,2,3,4,5,6,7,8,9,10,11,12));
assertEquals(list,inspector.getStructFieldsDataAsList(s1));
ListObjectInspector listOI=(ListObjectInspector)inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector();
assertEquals(ObjectInspector.Category.LIST,listOI.getCategory());
assertEquals(10,listOI.getListElement(list,10));
assertEquals(null,listOI.getListElement(list,-1));
assertEquals(null,listOI.getListElement(list,13));
assertEquals(13,listOI.getListLength(list));
Map map=new HashMap();
map.put(1,2);
map.put(2,4);
map.put(3,6);
MapObjectInspector mapOI=(MapObjectInspector)inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector();
assertEquals(3,mapOI.getMapSize(map));
assertEquals(4,mapOI.getMapValueElement(map,2));
}
InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcTimezone1 APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier ts=Lists.newArrayList();
ts.add("2003-01-01 01:00:00.000000222");
ts.add("1996-08-02 09:00:00.723100809");
ts.add("1999-01-01 02:00:00.999999999");
ts.add("1995-01-02 03:00:00.688888888");
ts.add("2002-01-01 04:00:00.1");
ts.add("2010-03-02 05:00:00.000009001");
ts.add("2005-01-01 06:00:00.000002229");
ts.add("2006-01-01 07:00:00.900203003");
ts.add("2003-01-01 08:00:00.800000007");
ts.add("1998-11-02 10:00:00.857340643");
ts.add("2008-10-02 11:00:00.0");
ts.add("2037-01-01 00:00:00.000999");
ts.add("2014-03-28 00:00:00.0");
for ( String t : ts) {
writer.addRow(Timestamp.valueOf(t));
}
writer.close();
TimeZone.setDefault(TimeZone.getTimeZone(readerTimeZone));
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(readerTimeZone,TimeZone.getDefault().getID());
RecordReader rows=reader.rows(null);
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
Timestamp got=((TimestampWritable)row).getTimestamp();
assertEquals(ts.get(idx++),got.toString());
}
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestOrcTimezone2 APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier ts=Lists.newArrayList();
ts.add("2003-01-01 01:00:00.000000222");
ts.add("1999-01-01 02:00:00.999999999");
ts.add("1995-01-02 03:00:00.688888888");
ts.add("2002-01-01 04:00:00.1");
ts.add("2010-03-02 05:00:00.000009001");
ts.add("2005-01-01 06:00:00.000002229");
ts.add("2006-01-01 07:00:00.900203003");
ts.add("2003-01-01 08:00:00.800000007");
ts.add("1996-08-02 09:00:00.723100809");
ts.add("1998-11-02 10:00:00.857340643");
ts.add("2008-10-02 11:00:00.0");
ts.add("2037-01-01 00:00:00.000999");
for ( String t : ts) {
writer.addRow(Timestamp.valueOf(t));
}
writer.close();
TimeZone.setDefault(TimeZone.getTimeZone(readerTimeZone));
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(readerTimeZone,TimeZone.getDefault().getID());
RecordReader rows=reader.rows(null);
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
Timestamp got=((TimestampWritable)row).getTimestamp();
assertEquals(ts.get(idx++),got.toString());
}
rows.close();
}
Class: org.apache.hadoop.hive.ql.io.orc.TestOrcWideTable EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestRLEv2 APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestRecordReaderImpl EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestStreamName InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestStringDictionary APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.orc.TestTypeDescription InternalCallVerifier EqualityVerifier ",struct.toString());
assertEquals("{\"category\": \"struct\", \"id\": 0, \"max\": 3, \"fields\": [\n" + " \"f1\": {\"category\": \"int\", \"id\": 1, \"max\": 1},\n" + " \"f2\": {\"category\": \"string\", \"id\": 2, \"max\": 2},\n"+ " \"f3\": {\"category\": \"decimal\", \"id\": 3, \"max\": 3, \"precision\": 38, \"scale\": 10}]}",struct.toJson());
struct=TypeDescription.createStruct().addField("f1",TypeDescription.createUnion().addUnionChild(TypeDescription.createByte()).addUnionChild(TypeDescription.createDecimal().withPrecision(20).withScale(10))).addField("f2",TypeDescription.createStruct().addField("f3",TypeDescription.createDate()).addField("f4",TypeDescription.createDouble()).addField("f5",TypeDescription.createBoolean())).addField("f6",TypeDescription.createChar().withMaxLength(100));
assertEquals("struct,f2:struct,f6:char(100)>",struct.toString());
assertEquals("{\"category\": \"struct\", \"id\": 0, \"max\": 8, \"fields\": [\n" + " \"f1\": {\"category\": \"union\", \"id\": 1, \"max\": 3, \"children\": [\n" + " {\"category\": \"tinyint\", \"id\": 2, \"max\": 2},\n"+ " {\"category\": \"decimal\", \"id\": 3, \"max\": 3, \"precision\": 20, \"scale\": 10}]},\n"+ " \"f2\": {\"category\": \"struct\", \"id\": 4, \"max\": 7, \"fields\": [\n"+ " \"f3\": {\"category\": \"date\", \"id\": 5, \"max\": 5},\n"+ " \"f4\": {\"category\": \"double\", \"id\": 6, \"max\": 6},\n"+ " \"f5\": {\"category\": \"boolean\", \"id\": 7, \"max\": 7}]},\n"+ " \"f6\": {\"category\": \"char\", \"id\": 8, \"max\": 8, \"length\": 100}]}",struct.toJson());
}
Class: org.apache.hadoop.hive.ql.io.orc.TestUnrolledBitPack APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier input=Lists.newArrayList(Longs.asList(inp));
Writer writer=OrcFile.createWriter(testFilePath,OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).compress(CompressionKind.NONE).bufferSize(10000));
for ( Long l : input) {
writer.addRow(l);
}
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows();
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(input.get(idx++).longValue(),((LongWritable)row).get());
}
}
Class: org.apache.hadoop.hive.ql.io.orc.TestVectorOrcFile APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier expectedList=expected.middle.list;
List actualList=(List)((OrcStruct)row.getFieldValue(9)).getFieldValue(0);
compareList(expectedList,actualList,"middle list " + i);
compareList(expected.list,(List)row.getFieldValue(10),"list " + i);
}
rows.close();
Iterator stripeIterator=reader.getStripes().iterator();
long offsetOfStripe2=0;
long offsetOfStripe4=0;
long lastRowOfStripe2=0;
for (int i=0; i < 5; ++i) {
StripeInformation stripe=stripeIterator.next();
if (i < 2) {
lastRowOfStripe2+=stripe.getNumberOfRows();
}
else if (i == 2) {
offsetOfStripe2=stripe.getOffset();
lastRowOfStripe2+=stripe.getNumberOfRows() - 1;
}
else if (i == 4) {
offsetOfStripe4=stripe.getOffset();
}
}
boolean[] columns=new boolean[reader.getStatistics().length];
columns[5]=true;
columns[9]=true;
rows=reader.rowsOptions(new Reader.Options().range(offsetOfStripe2,offsetOfStripe4 - offsetOfStripe2).include(columns));
rows.seekToRow(lastRowOfStripe2);
for (int i=0; i < 2; ++i) {
row=(OrcStruct)rows.next(row);
BigRow expected=createRandomRow(intValues,doubleValues,stringValues,byteValues,words,(int)(lastRowOfStripe2 + i));
assertEquals(expected.long1.longValue(),((LongWritable)row.getFieldValue(4)).get());
assertEquals(expected.string1,row.getFieldValue(8));
}
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier stats=reader.getStripeStatistics();
int numStripes=stats.size();
assertEquals(3,numStripes);
StripeStatistics ss1=stats.get(0);
StripeStatistics ss2=stats.get(1);
StripeStatistics ss3=stats.get(2);
assertEquals(5000,ss1.getColumnStatistics()[0].getNumberOfValues());
assertEquals(5000,ss2.getColumnStatistics()[0].getNumberOfValues());
assertEquals(1000,ss3.getColumnStatistics()[0].getNumberOfValues());
assertEquals(5000,(ss1.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(5000,(ss2.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(1000,(ss3.getColumnStatistics()[1]).getNumberOfValues());
assertEquals(1,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMinimum());
assertEquals(2,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMinimum());
assertEquals(3,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMinimum());
assertEquals(1,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMaximum());
assertEquals(2,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMaximum());
assertEquals(3,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMaximum());
assertEquals(5000,((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getSum());
assertEquals(10000,((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getSum());
assertEquals(3000,((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getSum());
assertEquals(5000,(ss1.getColumnStatistics()[2]).getNumberOfValues());
assertEquals(5000,(ss2.getColumnStatistics()[2]).getNumberOfValues());
assertEquals(1000,(ss3.getColumnStatistics()[2]).getNumberOfValues());
assertEquals("one",((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMinimum());
assertEquals("two",((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMinimum());
assertEquals("three",((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMinimum());
assertEquals("one",((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMaximum());
assertEquals("two",((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMaximum());
assertEquals("three",((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMaximum());
assertEquals(15000,((StringColumnStatistics)ss1.getColumnStatistics()[2]).getSum());
assertEquals(15000,((StringColumnStatistics)ss2.getColumnStatistics()[2]).getSum());
assertEquals(5000,((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum());
RecordReaderImpl recordReader=(RecordReaderImpl)reader.rows();
OrcProto.RowIndex[] index=recordReader.readRowIndex(0,null,null).getRowGroupIndex();
assertEquals(3,index.length);
List items=index[1].getEntryList();
assertEquals(1,items.size());
assertEquals(3,items.get(0).getPositionsCount());
assertEquals(0,items.get(0).getPositions(0));
assertEquals(0,items.get(0).getPositions(1));
assertEquals(0,items.get(0).getPositions(2));
assertEquals(1,items.get(0).getStatistics().getIntStatistics().getMinimum());
index=recordReader.readRowIndex(1,null,null).getRowGroupIndex();
assertEquals(3,index.length);
items=index[1].getEntryList();
assertEquals(2,items.get(0).getStatistics().getIntStatistics().getMaximum());
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier types=reader.getTypes();
assertEquals(3,types.size());
assertEquals(OrcProto.Type.Kind.STRUCT,types.get(0).getKind());
assertEquals(2,types.get(0).getSubtypesCount());
assertEquals(1,types.get(0).getSubtypes(0));
assertEquals(2,types.get(0).getSubtypes(1));
assertEquals(OrcProto.Type.Kind.INT,types.get(1).getKind());
assertEquals(0,types.get(1).getSubtypesCount());
assertEquals(OrcProto.Type.Kind.STRING,types.get(2).getKind());
assertEquals(0,types.get(2).getSubtypesCount());
RecordReader rows1=reader.rows(new boolean[]{true,true,false});
RecordReader rows2=reader.rows(new boolean[]{true,false,true});
r1=new Random(1);
r2=new Random(2);
OrcStruct row1=null;
OrcStruct row2=null;
for (int i=0; i < 21000; ++i) {
assertEquals(true,rows1.hasNext());
assertEquals(true,rows2.hasNext());
row1=(OrcStruct)rows1.next(row1);
row2=(OrcStruct)rows2.next(row2);
assertEquals(r1.nextInt(),((IntWritable)row1.getFieldValue(0)).get());
assertEquals(Long.toHexString(r2.nextLong()),row2.getFieldValue(1).toString());
}
assertEquals(false,rows1.hasNext());
assertEquals(false,rows2.hasNext());
rows1.close();
rows2.close();
}
APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>,list:array>,"+ "map:map>,ts:timestamp,"+ "decimal1:decimal(38,18)>",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BooleanObjectInspector bo=(BooleanObjectInspector)readerInspector.getStructFieldRef("boolean1").getFieldObjectInspector();
ByteObjectInspector by=(ByteObjectInspector)readerInspector.getStructFieldRef("byte1").getFieldObjectInspector();
ShortObjectInspector sh=(ShortObjectInspector)readerInspector.getStructFieldRef("short1").getFieldObjectInspector();
IntObjectInspector in=(IntObjectInspector)readerInspector.getStructFieldRef("int1").getFieldObjectInspector();
LongObjectInspector lo=(LongObjectInspector)readerInspector.getStructFieldRef("long1").getFieldObjectInspector();
FloatObjectInspector fl=(FloatObjectInspector)readerInspector.getStructFieldRef("float1").getFieldObjectInspector();
DoubleObjectInspector dbl=(DoubleObjectInspector)readerInspector.getStructFieldRef("double1").getFieldObjectInspector();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
StructObjectInspector mid=(StructObjectInspector)readerInspector.getStructFieldRef("middle").getFieldObjectInspector();
List midFields=mid.getAllStructFieldRefs();
ListObjectInspector midli=(ListObjectInspector)midFields.get(0).getFieldObjectInspector();
StructObjectInspector inner=(StructObjectInspector)midli.getListElementObjectInspector();
List inFields=inner.getAllStructFieldRefs();
ListObjectInspector li=(ListObjectInspector)readerInspector.getStructFieldRef("list").getFieldObjectInspector();
MapObjectInspector ma=(MapObjectInspector)readerInspector.getStructFieldRef("map").getFieldObjectInspector();
TimestampObjectInspector tso=(TimestampObjectInspector)readerInspector.getStructFieldRef("ts").getFieldObjectInspector();
HiveDecimalObjectInspector dco=(HiveDecimalObjectInspector)readerInspector.getStructFieldRef("decimal1").getFieldObjectInspector();
StringObjectInspector mk=(StringObjectInspector)ma.getMapKeyObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(false,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(1,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(1024,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(1.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-15.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
List midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
List list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(2,list.size());
assertEquals(3,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("good",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(4,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("bad",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
Map map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(0,map.size());
assertEquals(Timestamp.valueOf("2000-03-12 15:00:00"),tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(12))));
assertEquals(HiveDecimal.create("12345678.6547456"),dco.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(13))));
assertEquals(true,rows.hasNext());
rows.seekToRow(7499);
row=rows.next(null);
assertEquals(true,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(100,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(2048,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(2.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-5.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("bye",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(3,list.size());
assertEquals(100000000,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("cat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(-100000,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("in",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
assertEquals(1234,in.get(inner.getStructFieldData(list.get(2),inFields.get(0))));
assertEquals("hat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(2),inFields.get(1))));
map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(2,map.size());
boolean[] found=new boolean[2];
for ( Object key : map.keySet()) {
String str=mk.getPrimitiveJavaObject(key);
if (str.equals("chani")) {
assertEquals(false,found[0]);
assertEquals(5,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[0]=true;
}
else if (str.equals("mauddib")) {
assertEquals(false,found[1]);
assertEquals(1,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[1]=true;
}
else {
throw new IllegalArgumentException("Unknown key " + str);
}
}
assertEquals(true,found[0]);
assertEquals(true,found[1]);
assertEquals(Timestamp.valueOf("2000-03-12 15:00:01"),tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(12))));
assertEquals(HiveDecimal.create("12345678.6547457"),dco.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(13))));
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("foo",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("bar",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(bytes(0,1,2,3,4,5),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertNull(st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertNull(bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier tslist=Lists.newArrayList();
tslist.add(Timestamp.valueOf("2037-01-01 00:00:00.000999"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.000000222"));
tslist.add(Timestamp.valueOf("1999-01-01 00:00:00.999999999"));
tslist.add(Timestamp.valueOf("1995-01-01 00:00:00.688888888"));
tslist.add(Timestamp.valueOf("2002-01-01 00:00:00.1"));
tslist.add(Timestamp.valueOf("2010-03-02 00:00:00.000009001"));
tslist.add(Timestamp.valueOf("2005-01-01 00:00:00.000002229"));
tslist.add(Timestamp.valueOf("2006-01-01 00:00:00.900203003"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.800000007"));
tslist.add(Timestamp.valueOf("1996-08-02 00:00:00.723100809"));
tslist.add(Timestamp.valueOf("1998-11-02 00:00:00.857340643"));
tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
VectorizedRowBatch batch=new VectorizedRowBatch(1,1024);
LongColumnVector vec=new LongColumnVector(1024);
batch.cols[0]=vec;
batch.reset();
batch.size=tslist.size();
for (int i=0; i < tslist.size(); ++i) {
Timestamp ts=tslist.get(i);
vec.vector[i]=TimestampUtils.getTimeNanoSec(ts);
}
writer.addRowBatch(batch);
writer.close();
Reader reader=OrcFile.createReader(testFilePath,OrcFile.readerOptions(conf).filesystem(fs));
RecordReader rows=reader.rows(null);
int idx=0;
while (rows.hasNext()) {
Object row=rows.next(null);
assertEquals(tslist.get(idx++).getNanos(),((TimestampWritable)row).getNanos());
}
assertEquals(tslist.size(),rows.getRowNumber());
assertEquals(0,writer.getSchema().getMaximumId());
boolean[] expected=new boolean[]{false};
boolean[] included=OrcUtils.includeColumns("",writer.getSchema());
assertEquals(true,Arrays.equals(expected,included));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier ,decimal:decimal(38,18)>",inspector.getTypeName());
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-12 15:00:00")),row.getFieldValue(0));
OrcUnion union=(OrcUnion)row.getFieldValue(1);
assertEquals(0,union.getTag());
assertEquals(new IntWritable(42),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("12345678.6547456")),row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(2,rows.getRowNumber());
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(new Text("hello"),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")),row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
assertEquals(null,row.getFieldValue(1));
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
union=(OrcUnion)row.getFieldValue(1);
assertEquals(0,union.getTag());
assertEquals(null,union.getObject());
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(null,row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(null,union.getObject());
assertEquals(null,row.getFieldValue(2));
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf("1970-01-01 00:00:00")),row.getFieldValue(0));
assertEquals(new IntWritable(200000),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("10000000000000000000")),row.getFieldValue(2));
rand=new Random(42);
for (int i=1970; i < 2038; ++i) {
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf(i + "-05-05 12:34:56." + i)),row.getFieldValue(0));
if ((i & 1) == 0) {
assertEquals(0,union.getTag());
assertEquals(new IntWritable(i * i),union.getObject());
}
else {
assertEquals(1,union.getTag());
assertEquals(new Text(Integer.toString(i * i)),union.getObject());
}
assertEquals(new HiveDecimalWritable(HiveDecimal.create(new BigInteger(64,rand),rand.nextInt(18))),row.getFieldValue(2));
}
for (int i=0; i < 5000; ++i) {
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(1732050807),union.getObject());
}
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(0),union.getObject());
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(10),union.getObject());
row=(OrcStruct)rows.next(row);
assertEquals(new IntWritable(138),union.getObject());
assertEquals(false,rows.hasNext());
assertEquals(1.0,rows.getProgress(),0.00001);
assertEquals(reader.getNumberOfRows(),rows.getRowNumber());
rows.seekToRow(1);
row=(OrcStruct)rows.next(row);
assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),row.getFieldValue(0));
assertEquals(1,union.getTag());
assertEquals(new Text("hello"),union.getObject());
assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")),row.getFieldValue(2));
rows.close();
}
APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier >>,list:array>,"+ "map:map>>",readerInspector.getTypeName());
List fields=readerInspector.getAllStructFieldRefs();
BooleanObjectInspector bo=(BooleanObjectInspector)readerInspector.getStructFieldRef("boolean1").getFieldObjectInspector();
ByteObjectInspector by=(ByteObjectInspector)readerInspector.getStructFieldRef("byte1").getFieldObjectInspector();
ShortObjectInspector sh=(ShortObjectInspector)readerInspector.getStructFieldRef("short1").getFieldObjectInspector();
IntObjectInspector in=(IntObjectInspector)readerInspector.getStructFieldRef("int1").getFieldObjectInspector();
LongObjectInspector lo=(LongObjectInspector)readerInspector.getStructFieldRef("long1").getFieldObjectInspector();
FloatObjectInspector fl=(FloatObjectInspector)readerInspector.getStructFieldRef("float1").getFieldObjectInspector();
DoubleObjectInspector dbl=(DoubleObjectInspector)readerInspector.getStructFieldRef("double1").getFieldObjectInspector();
BinaryObjectInspector bi=(BinaryObjectInspector)readerInspector.getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st=(StringObjectInspector)readerInspector.getStructFieldRef("string1").getFieldObjectInspector();
StructObjectInspector mid=(StructObjectInspector)readerInspector.getStructFieldRef("middle").getFieldObjectInspector();
List midFields=mid.getAllStructFieldRefs();
ListObjectInspector midli=(ListObjectInspector)midFields.get(0).getFieldObjectInspector();
StructObjectInspector inner=(StructObjectInspector)midli.getListElementObjectInspector();
List inFields=inner.getAllStructFieldRefs();
ListObjectInspector li=(ListObjectInspector)readerInspector.getStructFieldRef("list").getFieldObjectInspector();
MapObjectInspector ma=(MapObjectInspector)readerInspector.getStructFieldRef("map").getFieldObjectInspector();
StringObjectInspector mk=(StringObjectInspector)ma.getMapKeyObjectInspector();
RecordReader rows=reader.rows();
Object row=rows.next(null);
assertNotNull(row);
assertEquals(false,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(1,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(1024,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(1.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-15.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(0,1,2,3,4),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("hi",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
List midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
List list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(2,list.size());
assertEquals(3,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("good",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(4,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("bad",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
Map map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(0,map.size());
assertEquals(true,rows.hasNext());
row=rows.next(row);
assertEquals(true,bo.get(readerInspector.getStructFieldData(row,fields.get(0))));
assertEquals(100,by.get(readerInspector.getStructFieldData(row,fields.get(1))));
assertEquals(2048,sh.get(readerInspector.getStructFieldData(row,fields.get(2))));
assertEquals(65536,in.get(readerInspector.getStructFieldData(row,fields.get(3))));
assertEquals(Long.MAX_VALUE,lo.get(readerInspector.getStructFieldData(row,fields.get(4))));
assertEquals(2.0,fl.get(readerInspector.getStructFieldData(row,fields.get(5))),0.00001);
assertEquals(-5.0,dbl.get(readerInspector.getStructFieldData(row,fields.get(6))),0.00001);
assertEquals(bytes(),bi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,fields.get(7))));
assertEquals("bye",st.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,fields.get(8))));
midRow=midli.getList(mid.getStructFieldData(readerInspector.getStructFieldData(row,fields.get(9)),midFields.get(0)));
assertNotNull(midRow);
assertEquals(2,midRow.size());
assertEquals(1,in.get(inner.getStructFieldData(midRow.get(0),inFields.get(0))));
assertEquals("bye",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(0),inFields.get(1))));
assertEquals(2,in.get(inner.getStructFieldData(midRow.get(1),inFields.get(0))));
assertEquals("sigh",st.getPrimitiveJavaObject(inner.getStructFieldData(midRow.get(1),inFields.get(1))));
list=li.getList(readerInspector.getStructFieldData(row,fields.get(10)));
assertEquals(3,list.size());
assertEquals(100000000,in.get(inner.getStructFieldData(list.get(0),inFields.get(0))));
assertEquals("cat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(0),inFields.get(1))));
assertEquals(-100000,in.get(inner.getStructFieldData(list.get(1),inFields.get(0))));
assertEquals("in",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(1),inFields.get(1))));
assertEquals(1234,in.get(inner.getStructFieldData(list.get(2),inFields.get(0))));
assertEquals("hat",st.getPrimitiveJavaObject(inner.getStructFieldData(list.get(2),inFields.get(1))));
map=ma.getMap(readerInspector.getStructFieldData(row,fields.get(11)));
assertEquals(2,map.size());
boolean[] found=new boolean[2];
for ( Object key : map.keySet()) {
String str=mk.getPrimitiveJavaObject(key);
if (str.equals("chani")) {
assertEquals(false,found[0]);
assertEquals(5,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[0]=true;
}
else if (str.equals("mauddib")) {
assertEquals(false,found[1]);
assertEquals(1,in.get(inner.getStructFieldData(map.get(key),inFields.get(0))));
assertEquals(str,st.getPrimitiveJavaObject(inner.getStructFieldData(map.get(key),inFields.get(1))));
found[1]=true;
}
else {
throw new IllegalArgumentException("Unknown key " + str);
}
}
assertEquals(true,found[0]);
assertEquals(true,found[1]);
assertEquals(false,rows.hasNext());
rows.close();
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.parquet.TestArrayCompatibility APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
}
Class: org.apache.hadoop.hive.ql.io.parquet.TestDataWritableWriter UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.parquet.TestHiveSchemaConverter APIUtilityVerifier InternalCallVerifier EqualityVerifier ";
final String hiveColumnNames="mapCol";
final List columnNames=createHiveColumnsFrom(hiveColumnNames);
final List columnTypes=createHiveTypeInfoFrom(hiveColumnTypes);
final MessageType messageTypeFound=HiveSchemaConverter.convert(columnNames,columnTypes);
assertEquals(1,messageTypeFound.getFieldCount());
org.apache.parquet.schema.Type topLevel=messageTypeFound.getFields().get(0);
assertEquals("mapCol",topLevel.getName());
assertEquals(OriginalType.MAP,topLevel.getOriginalType());
assertEquals(Repetition.OPTIONAL,topLevel.getRepetition());
assertEquals(1,topLevel.asGroupType().getFieldCount());
org.apache.parquet.schema.Type secondLevel=topLevel.asGroupType().getFields().get(0);
assertEquals("map",secondLevel.getName());
assertEquals(OriginalType.MAP_KEY_VALUE,secondLevel.getOriginalType());
assertEquals(Repetition.REPEATED,secondLevel.getRepetition());
}
Class: org.apache.hadoop.hive.ql.io.parquet.TestMapStructures APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("votes"),Arrays.asList("map"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("map_of_maps"),Arrays.asList("map>"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("examples"),Arrays.asList("map>"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("votes"),Arrays.asList("map"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("approx"),Arrays.asList("map>"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("examples"),Arrays.asList("map>"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier records=read(test);
Assert.assertEquals("Should have only one record",1,records.size());
assertEquals("Should match expected record",expected,records.get(0));
deserialize(records.get(0),Arrays.asList("matrix"),Arrays.asList("map,bigint>"));
}
Class: org.apache.hadoop.hive.ql.io.parquet.TestMapredParquetOutputFormat UtilityVerifier EqualityVerifier NullVerifier HybridVerifier outputFormat=(ParquetOutputFormat)mock(ParquetOutputFormat.class);
JobConf jobConf=new JobConf();
try {
new MapredParquetOutputFormat(outputFormat){
@Override protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper( ParquetOutputFormat realOutputFormat, JobConf jobConf, String finalOutPath, Progressable progress, Properties tableProperties) throws IOException {
assertEquals(outputFormat,realOutputFormat);
assertNotNull(jobConf.get(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA));
assertEquals("/foo",finalOutPath.toString());
assertEquals(mockProgress,progress);
throw new RuntimeException("passed tests");
}
}
.getHiveRecordWriter(jobConf,new Path("/foo"),null,false,tableProps,mockProgress);
fail("should throw runtime exception.");
}
catch ( RuntimeException e) {
assertEquals("passed tests",e.getMessage());
}
}
UtilityVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.parquet.TestParquetRecordReaderWrapper APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.parquet.TestParquetRowGroupFilter APIUtilityVerifier InternalCallVerifier EqualityVerifier children=Lists.newArrayList();
ExprNodeColumnDesc columnDesc=new ExprNodeColumnDesc(Integer.class,"intCol","T",false);
ExprNodeConstantDesc constantDesc=new ExprNodeConstantDesc(50);
children.add(columnDesc);
children.add(constantDesc);
ExprNodeGenericFuncDesc genericFuncDesc=new ExprNodeGenericFuncDesc(inspector,udf,children);
String searchArgumentStr=SerializationUtilities.serializeExpression(genericFuncDesc);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,searchArgumentStr);
ParquetRecordReaderWrapper recordReader=(ParquetRecordReaderWrapper)new MapredParquetInputFormat().getRecordReader(new FileSplit(testPath,0,fileLength(testPath),(String[])null),conf,null);
Assert.assertEquals("row group is not filtered correctly",1,recordReader.getFiltedBlocks().size());
constantDesc=new ExprNodeConstantDesc(100);
children.set(1,constantDesc);
genericFuncDesc=new ExprNodeGenericFuncDesc(inspector,udf,children);
searchArgumentStr=SerializationUtilities.serializeExpression(genericFuncDesc);
conf.set(TableScanDesc.FILTER_EXPR_CONF_STR,searchArgumentStr);
recordReader=(ParquetRecordReaderWrapper)new MapredParquetInputFormat().getRecordReader(new FileSplit(testPath,0,fileLength(testPath),(String[])null),conf,null);
Assert.assertEquals("row group is not filtered correctly",0,recordReader.getFiltedBlocks().size());
}
Class: org.apache.hadoop.hive.ql.io.parquet.read.TestParquetFilterPredicate APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.parquet.serde.TestAbstractParquetMapInspector InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier map=new HashMap();
map.put(new IntWritable(0),new IntWritable(1));
map.put(new IntWritable(2),new IntWritable(3));
map.put(new IntWritable(4),new IntWritable(5));
map.put(new IntWritable(6),new IntWritable(7));
assertEquals("Wrong size",4,inspector.getMapSize(map));
assertEquals("Wrong result of inspection",map,inspector.getMap(map));
}
InternalCallVerifier EqualityVerifier expected=new HashMap();
expected.put(new IntWritable(0),new IntWritable(1));
expected.put(new IntWritable(2),new IntWritable(3));
assertEquals("Wrong size",2,inspector.getMapSize(map));
assertEquals("Wrong result of inspection",expected,inspector.getMap(map));
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.parquet.serde.TestDeepParquetHiveMapInspector InternalCallVerifier EqualityVerifier map=new HashMap();
map.put(new IntWritable(0),new IntWritable(1));
map.put(new IntWritable(2),new IntWritable(3));
map.put(new IntWritable(4),new IntWritable(5));
map.put(new IntWritable(6),new IntWritable(7));
assertEquals("Wrong result of inspection",new IntWritable(1),inspector.getMapValueElement(map,new IntWritable(0)));
assertEquals("Wrong result of inspection",new IntWritable(3),inspector.getMapValueElement(map,new IntWritable(2)));
assertEquals("Wrong result of inspection",new IntWritable(5),inspector.getMapValueElement(map,new IntWritable(4)));
assertEquals("Wrong result of inspection",new IntWritable(7),inspector.getMapValueElement(map,new IntWritable(6)));
assertEquals("Wrong result of inspection",new IntWritable(1),inspector.getMapValueElement(map,new ShortWritable((short)0)));
assertEquals("Wrong result of inspection",new IntWritable(3),inspector.getMapValueElement(map,new ShortWritable((short)2)));
assertEquals("Wrong result of inspection",new IntWritable(5),inspector.getMapValueElement(map,new ShortWritable((short)4)));
assertEquals("Wrong result of inspection",new IntWritable(7),inspector.getMapValueElement(map,new ShortWritable((short)6)));
}
InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.parquet.serde.TestParquetHiveArrayInspector InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier expected=new ArrayList();
expected.add(new IntWritable(3));
expected.add(new IntWritable(5));
expected.add(new IntWritable(1));
assertEquals("Wrong size",3,inspector.getListLength(list));
assertEquals("Wrong result of inspection",expected,inspector.getList(list));
for (int i=0; i < expected.size(); ++i) {
assertEquals("Wrong result of inspection",expected.get(i),inspector.getListElement(list,i));
}
assertNull("Should be null",inspector.getListElement(list,3));
}
Class: org.apache.hadoop.hive.ql.io.parquet.serde.TestStandardParquetHiveMapInspector InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier map=new HashMap();
map.put(new IntWritable(0),new IntWritable(1));
map.put(new IntWritable(2),new IntWritable(3));
map.put(new IntWritable(4),new IntWritable(5));
map.put(new IntWritable(6),new IntWritable(7));
assertEquals("Wrong result of inspection",new IntWritable(1),inspector.getMapValueElement(map,new IntWritable(0)));
assertEquals("Wrong result of inspection",new IntWritable(3),inspector.getMapValueElement(map,new IntWritable(2)));
assertEquals("Wrong result of inspection",new IntWritable(5),inspector.getMapValueElement(map,new IntWritable(4)));
assertEquals("Wrong result of inspection",new IntWritable(7),inspector.getMapValueElement(map,new IntWritable(6)));
assertNull("Wrong result of inspection",inspector.getMapValueElement(map,new ShortWritable((short)0)));
assertNull("Wrong result of inspection",inspector.getMapValueElement(map,new ShortWritable((short)2)));
assertNull("Wrong result of inspection",inspector.getMapValueElement(map,new ShortWritable((short)4)));
assertNull("Wrong result of inspection",inspector.getMapValueElement(map,new ShortWritable((short)6)));
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.io.parquet.serde.primitive.TestParquetByteInspector EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.parquet.serde.primitive.TestParquetShortInspector InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.sarg.TestConvertAstToSearchArg APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 10 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 11 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 12 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 13 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 14 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 15 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 16 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 17 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 18 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " ";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(9,leaves.size());
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected="and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(" + "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 13)), lt(id, 16)), " + "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 13)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 13)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 14)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 14)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 14)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 15)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 15)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 15)), lt(id, 16))), "+ "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 13)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 13)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 13)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 14)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 14)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 14)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 15)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 15)), lt(id, 17))), "+ "or(or(or(lt(id, 18), lt(id, 12)), lt(id, 15)), lt(id, 17)))";
assertEquals(p.toString(),expected);
PredicateLeaf leaf=leaves.get(0);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(18L,leaf.getLiteral());
leaf=leaves.get(1);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(10L,leaf.getLiteral());
leaf=leaves.get(2);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(13L,leaf.getLiteral());
leaf=leaves.get(3);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(16L,leaf.getLiteral());
leaf=leaves.get(4);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(11L,leaf.getLiteral());
leaf=leaves.get(5);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(12L,leaf.getLiteral());
leaf=leaves.get(6);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(14L,leaf.getLiteral());
leaf=leaves.get(7);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(15L,leaf.getLiteral());
leaf=leaves.get(8);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(17L,leaf.getLiteral());
assertEquals("(and" + " (or leaf-0 leaf-1 leaf-2 leaf-3)" + " (or leaf-0 leaf-4 leaf-2 leaf-3)"+ " (or leaf-0 leaf-5 leaf-2 leaf-3)"+ " (or leaf-0 leaf-1 leaf-6 leaf-3)"+ " (or leaf-0 leaf-4 leaf-6 leaf-3)"+ " (or leaf-0 leaf-5 leaf-6 leaf-3)"+ " (or leaf-0 leaf-1 leaf-7 leaf-3)"+ " (or leaf-0 leaf-4 leaf-7 leaf-3)"+ " (or leaf-0 leaf-5 leaf-7 leaf-3)"+ " (or leaf-0 leaf-1 leaf-2 leaf-8)"+ " (or leaf-0 leaf-4 leaf-2 leaf-8)"+ " (or leaf-0 leaf-5 leaf-2 leaf-8)"+ " (or leaf-0 leaf-1 leaf-6 leaf-8)"+ " (or leaf-0 leaf-4 leaf-6 leaf-8)"+ " (or leaf-0 leaf-5 leaf-6 leaf-8)"+ " (or leaf-0 leaf-1 leaf-7 leaf-8)"+ " (or leaf-0 leaf-4 leaf-7 leaf-8)"+ " (or leaf-0 leaf-5 leaf-7 leaf-8))",sarg.getExpression().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 10 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 10 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " ";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(1,leaves.size());
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected="and(not(lt(id, 10)), not(lt(id, 10)))";
assertEquals(expected,p.toString());
assertEquals(PredicateLeaf.Type.LONG,leaves.get(0).getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaves.get(0).getOperator());
assertEquals("id",leaves.get(0).getColumnName());
assertEquals(10L,leaves.get(0).getLiteral());
assertEquals("(and (not leaf-0) (not leaf-0))",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
assertEquals(TruthValue.NO,sarg.evaluate(values(TruthValue.YES)));
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.NO)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NULL)));
assertEquals(TruthValue.NO_NULL,sarg.evaluate(values(TruthValue.YES_NULL)));
assertEquals(TruthValue.YES_NULL,sarg.evaluate(values(TruthValue.NO_NULL)));
assertEquals(TruthValue.YES_NO,sarg.evaluate(values(TruthValue.YES_NO)));
assertEquals(TruthValue.YES_NO_NULL,sarg.evaluate(values(TruthValue.YES_NO_NULL)));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 12 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " john \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " sue \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 34 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 50 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ "\n";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(3,leaves.size());
String[] conditions=new String[]{"not(eq(id, 12))","or(eq(first_name, Binary{\"john\"}), eq(first_name, Binary{\"sue\"}))","or(eq(id, 34), eq(id, 50))"};
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected=String.format("and(and(%1$s, %2$s), %3$s)",conditions);
assertEquals(expected,p.toString());
PredicateLeaf leaf=leaves.get(0);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.EQUALS,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(12L,leaf.getLiteral());
leaf=leaves.get(1);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.IN,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("john",leaf.getLiteralList().get(0));
assertEquals("sue",leaf.getLiteralList().get(1));
leaf=leaves.get(2);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.IN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(34L,leaf.getLiteralList().get(0));
assertEquals(50L,leaf.getLiteralList().get(1));
assertEquals("(and (not leaf-0) leaf-1 leaf-2)",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.YES)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NULL,TruthValue.YES,TruthValue.YES)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.NULL,TruthValue.YES)));
assertEquals(TruthValue.NO,sarg.evaluate(values(TruthValue.YES,TruthValue.YES,TruthValue.YES)));
assertEquals(TruthValue.NO,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.NO,sarg.evaluate(values(TruthValue.NO,TruthValue.YES_NULL,TruthValue.NO)));
assertEquals(TruthValue.NO_NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.NULL,TruthValue.YES_NO_NULL)));
assertEquals(TruthValue.NO_NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.NO_NULL)));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " owen \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " foobar \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " last_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 4 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " org.apache.hadoop.hive.ql.udf.UDFSubstr \n"+ " \n"+ " \n"+ " substr \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " false \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " david \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " greg \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(1,leaves.size());
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected="and(lt(first_name, Binary{\"greg\"}), not(lteq(first_name, Binary{\"david\"})))";
assertEquals(p.toString(),expected);
assertEquals(PredicateLeaf.Type.STRING,leaves.get(0).getType());
assertEquals(PredicateLeaf.Operator.BETWEEN,leaves.get(0).getOperator());
assertEquals("first_name",leaves.get(0).getColumnName());
assertEquals("leaf-0",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " sue \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 12 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 4 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(4,leaves.size());
String[] conditions=new String[]{"eq(first_name, null)","not(eq(first_name, Binary{\"sue\"}))","not(lt(id, 12))","lteq(id, 4)"};
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected=String.format("or(or(or(%1$s, %2$s), %3$s), %4$s)",conditions);
assertEquals(expected,p.toString());
PredicateLeaf leaf=leaves.get(0);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.IS_NULL,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals(null,leaf.getLiteral());
assertEquals(null,leaf.getLiteralList());
leaf=leaves.get(1);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.EQUALS,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("sue",leaf.getLiteral());
leaf=leaves.get(2);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(12L,leaf.getLiteral());
leaf=leaves.get(3);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(4L,leaf.getLiteral());
assertEquals("(or leaf-0 (not leaf-1) (not leaf-2) leaf-3)",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
assertEquals(TruthValue.NO,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.YES,TruthValue.YES,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.NO,TruthValue.NO,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.NO,TruthValue.NO)));
assertEquals(TruthValue.YES,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.YES,TruthValue.YES)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NULL,TruthValue.YES,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.NULL,TruthValue.YES,TruthValue.NO)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.NULL,TruthValue.NO)));
assertEquals(TruthValue.NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.YES,TruthValue.YES,TruthValue.NULL)));
assertEquals(TruthValue.YES_NO,sarg.evaluate(values(TruthValue.NO,TruthValue.YES_NO,TruthValue.YES,TruthValue.YES_NO)));
assertEquals(TruthValue.NO_NULL,sarg.evaluate(values(TruthValue.NO,TruthValue.YES_NULL,TruthValue.YES,TruthValue.NO_NULL)));
assertEquals(TruthValue.YES_NULL,sarg.evaluate(values(TruthValue.YES_NULL,TruthValue.YES_NO_NULL,TruthValue.YES,TruthValue.NULL)));
assertEquals(TruthValue.YES_NO_NULL,sarg.evaluate(values(TruthValue.NO_NULL,TruthValue.YES_NO_NULL,TruthValue.YES,TruthValue.NO)));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " false \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 23 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 45 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " alan \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " xxxxx \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 3 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " org.apache.hadoop.hive.ql.udf.UDFSubstr \n"+ " \n"+ " \n"+ " substr \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " smith \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " last_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 3 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " org.apache.hadoop.hive.ql.udf.UDFSubstr \n"+ " \n"+ " \n"+ " substr \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " yyy \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(3,leaves.size());
String[] conditions=new String[]{"lt(id, 45)","not(lteq(id, 23))","eq(first_name, Binary{\"alan\"})","eq(last_name, Binary{\"smith\"})"};
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; required binary last_name;}");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String expected=String.format("and(and(and(%1$s, %2$s), %3$s), %4$s)",conditions);
assertEquals(expected,p.toString());
PredicateLeaf leaf=leaves.get(0);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.BETWEEN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(null,leaf.getLiteral());
assertEquals(23L,leaf.getLiteralList().get(0));
assertEquals(45L,leaf.getLiteralList().get(1));
leaf=leaves.get(1);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.EQUALS,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("alan",leaf.getLiteral());
leaf=leaves.get(2);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.EQUALS,leaf.getOperator());
assertEquals("last_name",leaf.getColumnName());
assertEquals("smith",leaf.getLiteral());
assertEquals("(and leaf-0 leaf-1 leaf-2)",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " last_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " ";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(0,leaves.size());
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
assertNull(p);
assertEquals("YES_NO_NULL",sarg.getExpression().toString());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 1 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 3 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " true \n"+ " \n"+ " \n"+ " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n"+ " \n"+ " \n"+ " + \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 4 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " true \n"+ " \n"+ " \n"+ " org.apache.hadoop.hive.ql.udf.UDFOPPlus \n"+ " \n"+ " \n"+ " + \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " ";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(0,leaves.size());
assertEquals("YES_NO_NULL",sarg.getExpression().toString());
assertEquals(TruthValue.YES_NO_NULL,sarg.evaluate(values()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier \n" + " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " string \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " john \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " boolean \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " greg \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " alan \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " int \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 12 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 13 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 15 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 16 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " id \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " 30 \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " first_name \n"+ " \n"+ " \n"+ " orc_people \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " owen \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n"+ " \n";
SearchArgumentImpl sarg=(SearchArgumentImpl)ConvertAstToSearchArg.create(getFuncDesc(exprStr));
List leaves=sarg.getLeaves();
assertEquals(9,leaves.size());
MessageType schema=MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
FilterPredicate p=ParquetFilterPredicateConverter.toFilterPredicate(sarg,schema);
String[] conditions=new String[]{"eq(first_name, Binary{\"john\"})","not(lteq(first_name, Binary{\"greg\"}))","lt(first_name, Binary{\"alan\"})","not(lteq(id, 12))","not(lteq(id, 13))","lt(id, 15)","lt(id, 16)","eq(id, 30)","eq(first_name, Binary{\"owen\"})"};
String expected=String.format("and(or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %8$s), " + "or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %9$s))",conditions);
assertEquals(expected,p.toString());
PredicateLeaf leaf=leaves.get(0);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.EQUALS,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("john",leaf.getLiteral());
leaf=leaves.get(1);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("greg",leaf.getLiteral());
leaf=leaves.get(2);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("alan",leaf.getLiteral());
leaf=leaves.get(3);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(12L,leaf.getLiteral());
leaf=leaves.get(4);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(13L,leaf.getLiteral());
leaf=leaves.get(5);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(15L,leaf.getLiteral());
leaf=leaves.get(6);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.LESS_THAN,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(16L,leaf.getLiteral());
leaf=leaves.get(7);
assertEquals(PredicateLeaf.Type.LONG,leaf.getType());
assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS,leaf.getOperator());
assertEquals("id",leaf.getColumnName());
assertEquals(30L,leaf.getLiteral());
leaf=leaves.get(8);
assertEquals(PredicateLeaf.Type.STRING,leaf.getType());
assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS,leaf.getOperator());
assertEquals("first_name",leaf.getColumnName());
assertEquals("owen",leaf.getLiteral());
assertEquals("(and (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)" + " (not leaf-4) leaf-5 leaf-6 leaf-7)" + " (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)"+ " (not leaf-4) leaf-5 leaf-6 leaf-8))",sarg.getExpression().toString());
assertNoSharedNodes(sarg.getExpression(),Sets.newIdentityHashSet());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier newIdentityHashSet());
}
EqualityVerifier Class: org.apache.hadoop.hive.ql.lib.TestRuleRegExp UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier ns1=new Stack();
ns1.push(new TestNode(TableScanOperator.getOperatorName()));
ns1.push(new TestNode(FilterOperator.getOperatorName()));
Stack ns2=new Stack();
ns2.push(new TestNode(TableScanOperator.getOperatorName()));
ns2.push(new TestNode(FileSinkOperator.getOperatorName()));
try {
assertNotEquals(rule1.cost(ns1),-1);
assertNotEquals(rule1.cost(ns2),-1);
}
catch ( SemanticException e) {
fail(e.getMessage());
}
Stack ns3=new Stack();
ns3.push(new TestNode(ReduceSinkOperator.getOperatorName()));
ns3.push(new TestNode(ReduceSinkOperator.getOperatorName()));
ns3.push(new TestNode(FileSinkOperator.getOperatorName()));
try {
assertEquals(rule1.cost(ns3),-1);
}
catch ( SemanticException e) {
fail(e.getMessage());
}
}
UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier ns1=new Stack();
ns1.push(new TestNode(ReduceSinkOperator.getOperatorName()));
ns1.push(new TestNode(SelectOperator.getOperatorName()));
ns1.push(new TestNode(FileSinkOperator.getOperatorName()));
try {
assertEquals(rule1.cost(ns1),patternStr.length());
}
catch ( SemanticException e) {
fail(e.getMessage());
}
Stack ns2=new Stack();
ns2.push(new TestNode(ReduceSinkOperator.getOperatorName()));
ns1.push(new TestNode(TableScanOperator.getOperatorName()));
ns1.push(new TestNode(FileSinkOperator.getOperatorName()));
try {
assertEquals(rule1.cost(ns2),-1);
}
catch ( SemanticException e) {
fail(e.getMessage());
}
}
Class: org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.rollbackTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(3,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
txnMgr.releaseLocks(locks);
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
ShowLocksResponse rsp=((DbLockManager)txnMgr.getLockManager()).getLocks();
List elms=rsp.getLocks();
Assert.assertEquals(1,elms.size());
Assert.assertNotNull(elms.get(0).getTablename());
Assert.assertNull(elms.get(0).getPartname());
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(4,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.commitTxn();
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(1,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
BooleanVerifier EqualityVerifier NullVerifier HybridVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier locks=ctx.getHiveLocks();
Assert.assertEquals(1,locks.size());
Assert.assertEquals(4,TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock)locks.get(0)).lockId));
txnMgr.getLockManager().unlock(locks.get(0));
locks=txnMgr.getLockManager().getLocks(false,false);
Assert.assertEquals(0,locks.size());
}
Class: org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2 EqualityVerifier locks=getLocks();
Assert.assertEquals("Unexpected lock count",2,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T5",null,locks.get(0));
checkLock(LockType.EXCLUSIVE,LockState.ACQUIRED,"default","T4",null,locks.get(1));
txnMgr.getLockManager().releaseLocks(ctx.getHiveLocks());
Assert.assertEquals("Lock remained",0,getLocks().size());
cpr=driver.run("drop table if exists T5");
checkCmdOnDriver(cpr);
cpr=driver.run("drop table if exists T4");
checkCmdOnDriver(cpr);
}
EqualityVerifier locks=getLocks();
Assert.assertEquals("Unexpected lock count",2,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T2",null,locks.get(0));
checkLock(LockType.EXCLUSIVE,LockState.ACQUIRED,"default","T3",null,locks.get(1));
txnMgr.getLockManager().releaseLocks(ctx.getHiveLocks());
Assert.assertEquals("Lock remained",0,getLocks().size());
cpr=driver.run("drop table if exists T1");
checkCmdOnDriver(cpr);
cpr=driver.run("drop table if exists T2");
checkCmdOnDriver(cpr);
}
EqualityVerifier updateLocks=ctx.getHiveLocks();
cpr=driver.compileAndRespond("drop database if exists temp");
LockState lockState=((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(),ctx,"Fiddler",false);
List locks=getLocks();
Assert.assertEquals("Unexpected lock count",2,locks.size());
checkLock(LockType.SHARED_WRITE,LockState.ACQUIRED,"temp","T7",null,locks.get(0));
checkLock(LockType.EXCLUSIVE,LockState.WAITING,"temp",null,null,locks.get(1));
txnMgr.getLockManager().releaseLocks(updateLocks);
lockState=((DbLockManager)txnMgr.getLockManager()).checkLock(locks.get(1).getLockid());
locks=getLocks();
Assert.assertEquals("Unexpected lock count",1,locks.size());
checkLock(LockType.EXCLUSIVE,LockState.ACQUIRED,"temp",null,null,locks.get(0));
List xLock=new ArrayList(0);
xLock.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
txnMgr.getLockManager().releaseLocks(xLock);
}
EqualityVerifier locks=getLocks();
Assert.assertEquals("Unexpected lock count",1,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default",null,null,locks.get(0));
txnMgr.getLockManager().releaseLocks(ctx.getHiveLocks());
Assert.assertEquals("Lock remained",0,getLocks().size());
}
EqualityVerifier deleteLocks=ctx.getHiveLocks();
cpr=driver.compileAndRespond("select a from T8");
checkCmdOnDriver(cpr);
txnMgr.acquireLocks(driver.getPlan(),ctx,"Fiddler");
cpr=driver.compileAndRespond("update T8 set a = 1 where b = 1");
checkCmdOnDriver(cpr);
LockState lockState=((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(),ctx,"Practical",false);
List locks=getLocks();
Assert.assertEquals("Unexpected lock count",3,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T8",null,locks.get(0));
checkLock(LockType.SHARED_WRITE,LockState.ACQUIRED,"default","T8",null,locks.get(1));
checkLock(LockType.SHARED_WRITE,LockState.WAITING,"default","T8",null,locks.get(2));
txnMgr.getLockManager().releaseLocks(deleteLocks);
lockState=((DbLockManager)txnMgr.getLockManager()).checkLock(locks.get(2).getLockid());
locks=getLocks();
Assert.assertEquals("Unexpected lock count",2,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T8",null,locks.get(0));
checkLock(LockType.SHARED_WRITE,LockState.ACQUIRED,"default","T8",null,locks.get(1));
List relLocks=new ArrayList(2);
relLocks.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
relLocks.add(new DbLockManager.DbHiveLock(locks.get(1).getLockid()));
txnMgr.getLockManager().releaseLocks(relLocks);
cpr=driver.run("drop table if exists T6");
locks=getLocks();
Assert.assertEquals("Unexpected number of locks found",0,locks.size());
checkCmdOnDriver(cpr);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier locks=getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count",1,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T9",null,locks.get(0));
cpr=driver.compileAndRespond("drop table T9");
checkCmdOnDriver(cpr);
try {
otherTxnMgr.acquireLocks(driver.getPlan(),ctx,"Winston Winnfield");
}
catch ( LockException ex) {
Assert.assertEquals("Got wrong lock exception",ErrorMsg.LOCK_ACQUIRE_TIMEDOUT,ex.getCanonicalErrorMsg());
}
locks=getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count",1,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T9",null,locks.get(0));
otherTxnMgr.closeTxnManager();
}
EqualityVerifier selectLocks=ctx.getHiveLocks();
cpr=driver.compileAndRespond("drop table if exists T6");
checkCmdOnDriver(cpr);
LockState lockState=((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(),ctx,"Fiddler",false);
List locks=getLocks();
Assert.assertEquals("Unexpected lock count",2,locks.size());
checkLock(LockType.SHARED_READ,LockState.ACQUIRED,"default","T6",null,locks.get(0));
checkLock(LockType.EXCLUSIVE,LockState.WAITING,"default","T6",null,locks.get(1));
txnMgr.getLockManager().releaseLocks(selectLocks);
lockState=((DbLockManager)txnMgr.getLockManager()).checkLock(locks.get(1).getLockid());
locks=getLocks();
Assert.assertEquals("Unexpected lock count",1,locks.size());
checkLock(LockType.EXCLUSIVE,LockState.ACQUIRED,"default","T6",null,locks.get(0));
List xLock=new ArrayList(0);
xLock.add(new DbLockManager.DbHiveLock(locks.get(0).getLockid()));
txnMgr.getLockManager().releaseLocks(xLock);
cpr=driver.run("drop table if exists T6");
locks=getLocks();
Assert.assertEquals("Unexpected number of locks found",0,locks.size());
checkCmdOnDriver(cpr);
}
Class: org.apache.hadoop.hive.ql.lockmgr.TestDummyTxnManager InternalCallVerifier EqualityVerifier expectedLocks=new ArrayList();
expectedLocks.add(new ZooKeeperHiveLock("default",new HiveLockObject(),HiveLockMode.SHARED));
expectedLocks.add(new ZooKeeperHiveLock("default.table1",new HiveLockObject(),HiveLockMode.SHARED));
when(mockLockManager.lock(anyListOf(HiveLockObj.class),eq(false))).thenReturn(expectedLocks);
doNothing().when(mockLockManager).setContext(any(HiveLockManagerCtx.class));
doNothing().when(mockLockManager).close();
ArgumentCaptor lockObjsCaptor=ArgumentCaptor.forClass(List.class);
when(mockQueryPlan.getInputs()).thenReturn(createReadEntities());
when(mockQueryPlan.getOutputs()).thenReturn(new HashSet());
txnMgr.acquireLocks(mockQueryPlan,ctx,"fred");
Assert.assertEquals("db1",SessionState.get().getCurrentDatabase());
List resultLocks=ctx.getHiveLocks();
Assert.assertEquals(expectedLocks.size(),resultLocks.size());
Assert.assertEquals(expectedLocks.get(0).getHiveLockMode(),resultLocks.get(0).getHiveLockMode());
Assert.assertEquals(expectedLocks.get(0).getHiveLockObject().getName(),resultLocks.get(0).getHiveLockObject().getName());
Assert.assertEquals(expectedLocks.get(1).getHiveLockMode(),resultLocks.get(1).getHiveLockMode());
Assert.assertEquals(expectedLocks.get(0).getHiveLockObject().getName(),resultLocks.get(0).getHiveLockObject().getName());
verify(mockLockManager).lock(lockObjsCaptor.capture(),eq(false));
List lockObjs=lockObjsCaptor.getValue();
Assert.assertEquals(2,lockObjs.size());
Assert.assertEquals("default",lockObjs.get(0).getName());
Assert.assertEquals(HiveLockMode.SHARED,lockObjs.get(0).mode);
Assert.assertEquals("default/table1",lockObjs.get(1).getName());
Assert.assertEquals(HiveLockMode.SHARED,lockObjs.get(1).mode);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier lockObjs=new ArrayList();
String path1="path1";
String path2="path2";
HiveLockObjectData lockData1=new HiveLockObjectData("query1","1","IMPLICIT","drop table table1");
HiveLockObjectData lockData2=new HiveLockObjectData("query1","1","IMPLICIT","drop table table1");
lockObjs.add(new HiveLockObj(new HiveLockObject(path1,lockData1),HiveLockMode.SHARED));
String name1=lockObjs.get(lockObjs.size() - 1).getName();
lockObjs.add(new HiveLockObj(new HiveLockObject(path1,lockData1),HiveLockMode.EXCLUSIVE));
lockObjs.add(new HiveLockObj(new HiveLockObject(path2,lockData2),HiveLockMode.SHARED));
String name2=lockObjs.get(lockObjs.size() - 1).getName();
lockObjs.add(new HiveLockObj(new HiveLockObject(path2,lockData2),HiveLockMode.SHARED));
lockObjs.add(new HiveLockObj(new HiveLockObject(path2,lockData2),HiveLockMode.SHARED));
DummyTxnManager.dedupLockObjects(lockObjs);
Assert.assertEquals("Locks should be deduped",2,lockObjs.size());
Comparator cmp=new Comparator(){
@Override public int compare( HiveLockObj lock1, HiveLockObj lock2){
return lock1.getName().compareTo(lock2.getName());
}
}
;
Collections.sort(lockObjs,cmp);
HiveLockObj lockObj=lockObjs.get(0);
Assert.assertEquals(name1,lockObj.getName());
Assert.assertEquals(HiveLockMode.EXCLUSIVE,lockObj.getMode());
lockObj=lockObjs.get(1);
Assert.assertEquals(name2,lockObj.getName());
Assert.assertEquals(HiveLockMode.SHARED,lockObj.getMode());
}
Class: org.apache.hadoop.hive.ql.lockmgr.TestHiveLockObject InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.lockmgr.zookeeper.TestZookeeperLockManager InternalCallVerifier EqualityVerifier APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.log.TestLog4j2Appenders InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.optimizer.TestGenMapRedUtilsUsePartitionColumnsPositive APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.optimizer.calcite.TestCBORuleFiredOnlyOnce EqualityVerifier of(DummyRule.INSTANCE));
HiveRulesRegistry registry=new HiveRulesRegistry();
HivePlannerContext context=new HivePlannerContext(null,registry);
HepPlanner planner=new HepPlanner(programBuilder.build(),context);
RexBuilder rexBuilder=new RexBuilder(new JavaTypeFactoryImpl());
RelOptCluster cluster=RelOptCluster.create(planner,rexBuilder);
HiveDefaultRelMetadataProvider mdProvider=new HiveDefaultRelMetadataProvider(conf);
List list=Lists.newArrayList();
list.add(mdProvider.getMetadataProvider());
planner.registerMetadataProviders(list);
RelMetadataProvider chainedProvider=ChainedRelMetadataProvider.of(list);
final RelNode node=new DummyNode(cluster,cluster.traitSet());
node.getCluster().setMetadataProvider(new CachingRelMetadataProvider(chainedProvider,planner));
planner.setRoot(node);
planner.findBestExp();
assertEquals(3,DummyRule.INSTANCE.numberMatches);
assertEquals(1,DummyRule.INSTANCE.numberOnMatch);
}
Class: org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.TestDynamicMultiDimeCollection APIUtilityVerifier EqualityVerifier > uniqSkewedElements=new ArrayList>();
List v1=Arrays.asList("1","2");
List v2=Arrays.asList("3","4");
uniqSkewedElements.add(v1);
uniqSkewedElements.add(v2);
List> actual=ListBucketingPruner.DynamicMultiDimensionalCollection.flat(uniqSkewedElements);
List> expected=new ArrayList>();
v1=Arrays.asList("1","3");
v2=Arrays.asList("1","4");
List v3=Arrays.asList("2","3");
List v4=Arrays.asList("2","4");
expected.add(v1);
expected.add(v2);
expected.add(v3);
expected.add(v4);
Assert.assertEquals(expected,actual);
}
APIUtilityVerifier EqualityVerifier > values=new ArrayList>();
List v1=Arrays.asList("1","a","x");
values.add(v1);
List> actuals=ListBucketingPruner.DynamicMultiDimensionalCollection.uniqueElementsList(values,DEF_DIR);
List> expecteds=new ArrayList>();
v1=Arrays.asList("1","default");
List v2=Arrays.asList("a","default");
List v3=Arrays.asList("x","default");
expecteds.add(v1);
expecteds.add(v2);
expecteds.add(v3);
Assert.assertEquals(expecteds,actuals);
}
APIUtilityVerifier EqualityVerifier > values=new ArrayList>();
List v1=Arrays.asList("1");
List v2=Arrays.asList("2");
List v3=Arrays.asList("3");
List v4=Arrays.asList("4");
values.add(v1);
values.add(v2);
values.add(v3);
values.add(v4);
List> actuals=ListBucketingPruner.DynamicMultiDimensionalCollection.uniqueElementsList(values,DEF_DIR);
List> expecteds=new ArrayList>();
v1=Arrays.asList("1","2","3","4","default");
expecteds.add(v1);
Assert.assertEquals(expecteds,actuals);
}
APIUtilityVerifier EqualityVerifier > values=new ArrayList>();
List v1=Arrays.asList("1","a","x");
List v2=Arrays.asList("2","b","x");
List v3=Arrays.asList("1","c","x");
List v4=Arrays.asList("2","a","y");
values.add(v1);
values.add(v2);
values.add(v3);
values.add(v4);
List> actuals=ListBucketingPruner.DynamicMultiDimensionalCollection.uniqueElementsList(values,DEF_DIR);
List> expecteds=new ArrayList>();
v1=Arrays.asList("1","2","default");
v2=Arrays.asList("a","b","c","default");
v3=Arrays.asList("x","y","default");
expecteds.add(v1);
expecteds.add(v2);
expecteds.add(v3);
Assert.assertEquals(expecteds,actuals);
}
APIUtilityVerifier EqualityVerifier > uniqSkewedElements=new ArrayList>();
List v1=Arrays.asList("1","2");
uniqSkewedElements.add(v1);
List> actual=ListBucketingPruner.DynamicMultiDimensionalCollection.flat(uniqSkewedElements);
List> expected=new ArrayList>();
v1=Arrays.asList("1");
List v2=Arrays.asList("2");
expected.add(v1);
expected.add(v2);
Assert.assertEquals(expected,actual);
}
Class: org.apache.hadoop.hive.ql.optimizer.physical.TestVectorizer BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier params=new ArrayList();
params.add(exprNodeDesc);
aggDesc.setParameters(params);
GenericUDFAbs absUdf=new GenericUDFAbs();
exprNodeDesc.setGenericUDF(absUdf);
List children=new ArrayList();
ExprNodeColumnDesc colExprA=new ExprNodeColumnDesc(Integer.class,"col1","T",false);
ExprNodeColumnDesc colExprB=new ExprNodeColumnDesc(Integer.class,"col2","T",false);
children.add(colExprA);
exprNodeDesc.setChildren(children);
ArrayList outputColumnNames=new ArrayList();
outputColumnNames.add("_col0");
GroupByDesc desc=new GroupByDesc();
desc.setOutputColumnNames(outputColumnNames);
ArrayList aggDescList=new ArrayList();
aggDescList.add(aggDesc);
desc.setAggregators(aggDescList);
ArrayList grpByKeys=new ArrayList();
grpByKeys.add(colExprB);
desc.setKeys(grpByKeys);
GroupByOperator gbyOp=new GroupByOperator(new CompilationOpContext());
gbyOp.setConf(desc);
Vectorizer v=new Vectorizer();
Assert.assertTrue(v.validateMapWorkOperator(gbyOp,null,false));
VectorGroupByOperator vectorOp=(VectorGroupByOperator)v.vectorizeOperator(gbyOp,vContext,false);
Assert.assertEquals(VectorUDAFSumLong.class,vectorOp.getAggregators()[0].getClass());
VectorUDAFSumLong udaf=(VectorUDAFSumLong)vectorOp.getAggregators()[0];
Assert.assertEquals(FuncAbsLongToLong.class,udaf.getInputExpression().getClass());
}
Class: org.apache.hadoop.hive.ql.optimizer.ppr.TestPositivePartitionPrunerCompactExpr APIUtilityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.TestColumnAccess APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=columnAccessInfo.getTableToColumnAccessMap().get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
cols=columnAccessInfo.getTableToColumnAccessMap().get("default@t2");
Assert.assertNotNull(cols);
Assert.assertEquals(3,cols.size());
Assert.assertNotNull(cols.contains("id2"));
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
Map> tableColsMap=getColsFromReadEntity(plan.getInputs());
cols=tableColsMap.get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
cols=tableColsMap.get("default@t2");
Assert.assertNotNull(cols);
Assert.assertEquals(3,cols.size());
Assert.assertNotNull(cols.contains("id2"));
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=columnAccessInfo.getTableToColumnAccessMap().get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
Map> tableColsMap=getColsFromReadEntity(plan.getInputs());
cols=tableColsMap.get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=columnAccessInfo.getTableToColumnAccessMap().get("default@v1");
Assert.assertNull(cols);
cols=columnAccessInfo.getTableToColumnAccessMap().get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
cols=columnAccessInfo.getTableToColumnAccessMap().get("default@t2");
Assert.assertNotNull(cols);
Assert.assertEquals(3,cols.size());
Assert.assertNotNull(cols.contains("id2"));
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
Map> tableColsMap=getColsFromReadEntity(plan.getInputs());
cols=tableColsMap.get("default@v1");
Assert.assertNull(cols);
cols=tableColsMap.get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2,cols.size());
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
cols=tableColsMap.get("default@t2");
Assert.assertNotNull(cols);
Assert.assertEquals(3,cols.size());
Assert.assertNotNull(cols.contains("id2"));
Assert.assertNotNull(cols.contains("id1"));
Assert.assertNotNull(cols.contains("name1"));
}
Class: org.apache.hadoop.hive.ql.parse.TestGenTezWork BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.TestHiveDecimalParse APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestIUD EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestParseDriver IterativeVerifier InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestQBCompact InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier parts=desc.getPartSpec();
Assert.assertEquals(1,parts.size());
Assert.assertEquals("today",parts.get("ds"));
}
InternalCallVerifier EqualityVerifier parts=desc.getPartSpec();
Assert.assertEquals(1,parts.size());
Assert.assertEquals("today",parts.get("ds"));
}
BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.TestQBJoinTreeApplyPredicate APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestQBSubQuery APIUtilityVerifier InternalCallVerifier EqualityVerifier sqs=SubQueryUtils.findSubQueries((ASTNode)where.getChild(0));
ASTNode sq=sqs.get(0);
ASTNode newWhere=SubQueryUtils.rewriteParentQueryWhere((ASTNode)where.getChild(0),sq);
Assert.assertEquals("(= 1 1)",newWhere.toStringTree());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier sqs=SubQueryUtils.findSubQueries((ASTNode)where.getChild(0));
Assert.assertEquals(1,sqs.size());
ASTNode sq=sqs.get(0);
Assert.assertEquals("(tok_subquery_expr (tok_subquery_op in) (tok_query (tok_from (tok_tabref (tok_tabname src) s1)) (tok_insert (tok_destination (tok_dir tok_tmp_file)) (tok_select (tok_selexpr (tok_table_or_col key))) (tok_where (and (> (. (tok_table_or_col s1) key) '9') (> (. (tok_table_or_col s1) value) '9'))))) (. (tok_table_or_col src) key))",sq.toStringTree());
}
EqualityVerifier sqs=SubQueryUtils.findSubQueries((ASTNode)where.getChild(0));
ASTNode sq=sqs.get(0);
ASTNode sqWhere=where((ASTNode)sq.getChild(1));
List conjuncts=new ArrayList();
SubQueryUtils.extractConjuncts((ASTNode)sqWhere.getChild(0),conjuncts);
Assert.assertEquals(2,conjuncts.size());
Assert.assertEquals("(> (. (tok_table_or_col s1) key) '9')",conjuncts.get(0).toStringTree());
Assert.assertEquals("(> (. (tok_table_or_col s1) value) '9')",conjuncts.get(1).toStringTree());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier sqs=SubQueryUtils.findSubQueries((ASTNode)where.getChild(0));
ASTNode sq=sqs.get(0);
ASTNode newWhere=SubQueryUtils.rewriteParentQueryWhere((ASTNode)where.getChild(0),sq);
Assert.assertEquals("(> (tok_table_or_col value) '9')",newWhere.toStringTree());
}
Class: org.apache.hadoop.hive.ql.parse.TestSQL11ReservedKeyWordsNegative BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.TestSQL11ReservedKeyWordsPositive EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestSplitSample EqualityVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.TestUnpermittedCharsInColumnNameCreateTableNegative BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.authorization.TestHiveAuthorizationTaskFactory BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.parse.authorization.TestSessionUserName InternalCallVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.parse.authorization.plugin.sqlstd.TestOperation2Privilege BranchVerifier UtilityVerifier EqualityVerifier HybridVerifier operationMapKeys=Operation2Privilege.getOperationTypes();
for ( HiveOperationType operationType : HiveOperationType.values()) {
if (!operationMapKeys.contains(operationType)) {
fail("Unable to find corresponding entry in Operation2Privilege map for HiveOperationType " + operationType);
}
}
assertEquals("Check if Operation2Privilege, HiveOperationType have same number of instances",operationMapKeys.size(),HiveOperationType.values().length);
}
Class: org.apache.hadoop.hive.ql.parse.positive.TestTransactionStatement EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.plan.TestConditionalResolverCommonJoin InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier > pathToAliases=new HashMap>();
pathToAliases.put("path1",new ArrayList(Arrays.asList("alias1","alias2")));
pathToAliases.put("path2",new ArrayList(Arrays.asList("alias3")));
HashMap aliasToKnownSize=new HashMap();
aliasToKnownSize.put("alias1",1024l);
aliasToKnownSize.put("alias2",2048l);
aliasToKnownSize.put("alias3",4096l);
DDLTask task1=new DDLTask();
task1.setId("alias2");
DDLTask task2=new DDLTask();
task2.setId("alias3");
HashMap,Set> taskToAliases=new LinkedHashMap,Set>();
taskToAliases.put(task1,new HashSet(Arrays.asList("alias2")));
taskToAliases.put(task2,new HashSet(Arrays.asList("alias3")));
ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx ctx=new ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx();
ctx.setPathToAliases(pathToAliases);
ctx.setTaskToAliases(taskToAliases);
ctx.setAliasToKnownSize(aliasToKnownSize);
HiveConf conf=new HiveConf();
conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE,4096);
Task resolved=resolver.resolveMapJoinTask(ctx,conf);
Assert.assertEquals("alias3",resolved.getId());
conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE,65536);
resolved=resolver.resolveMapJoinTask(ctx,conf);
Assert.assertEquals("alias3",resolved.getId());
conf.setLongVar(HiveConf.ConfVars.HIVESMALLTABLESFILESIZE,2048);
resolved=resolver.resolveMapJoinTask(ctx,conf);
Assert.assertNull(resolved);
}
Class: org.apache.hadoop.hive.ql.plan.TestCreateMacroDesc InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.plan.TestDropMacroDesc EqualityVerifier Class: org.apache.hadoop.hive.ql.plan.TestReadEntityDirect BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.plan.TestTezWork APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier IterativeVerifier InternalCallVerifier EqualityVerifier sorted=work.getAllWork();
for (int i=0; i < 5; ++i) {
Assert.assertEquals(sorted.get(i),nodes.get(4 - i));
}
}
InternalCallVerifier EqualityVerifier (){
@Override public Void answer( InvocationOnMock invocation) throws Throwable {
conf.unset(MR_JAR_PROPERTY);
return null;
}
}
).when(baseWork).configureJobConf(conf);
work.add(baseWork);
work.configureJobConfAndExtractJars(conf);
Assert.assertEquals("file:///tmp/foo1.jar",conf.get(MR_JAR_PROPERTY));
}
InternalCallVerifier EqualityVerifier (){
@Override public Void answer( InvocationOnMock invocation) throws Throwable {
conf.setStrings(MR_JAR_PROPERTY,"file:///tmp/foo1.jar","file:///tmp/foo2.jar");
return null;
}
}
).when(baseWork).configureJobConf(conf);
work.add(baseWork);
work.configureJobConfAndExtractJars(conf);
Assert.assertEquals("file:///tmp/foo1.jar,file:///tmp/foo2.jar",conf.get(MR_JAR_PROPERTY));
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier (){
@Override public Void answer( InvocationOnMock invocation) throws Throwable {
conf.set(MR_JAR_PROPERTY,"file:///tmp/foo2.jar");
return null;
}
}
).when(baseWork).configureJobConf(conf);
work.add(baseWork);
work.configureJobConfAndExtractJars(conf);
Assert.assertEquals("file:///tmp/foo1.jar,file:///tmp/foo2.jar",conf.get(MR_JAR_PROPERTY));
}
Class: org.apache.hadoop.hive.ql.plan.TestViewEntity BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.processors.TestCommandProcessorFactory APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.processors.TestCompileProcessor BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.security.FolderPermissionBase BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.security.TestMultiAuthorizationPreEventListener EqualityVerifier authCalls=DummyHiveMetastoreAuthorizationProvider.authCalls;
int listSize=0;
assertEquals(listSize,authCalls.size());
driver.run("create database " + dbName);
listSize=2;
assertEquals(listSize,authCalls.size());
Database db=msc.getDatabase(dbName);
listSize+=2;
Database dbFromEvent=(Database)assertAndExtractSingleObjectFromEvent(listSize,authCalls,DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.DB);
validateCreateDb(db,dbFromEvent);
}
Class: org.apache.hadoop.hive.ql.security.TestPasswordWithConfig InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.security.TestPasswordWithCredentialProvider InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.security.TestStorageBasedMetastoreAuthorizationDrops InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.security.authorization.plugin.TestHiveAuthorizerCheckInvocation APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier inputs=getHivePrivilegeObjectInputs().getLeft();
checkSingleTableInput(inputs);
HivePrivilegeObject tableObj=inputs.get(0);
assertNull("columns used",tableObj.getColumns());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier outputs=getHivePrivilegeObjectInputs().getRight();
HivePrivilegeObject funcObj=outputs.get(0);
assertEquals("input type",HivePrivilegeObjectType.FUNCTION,funcObj.getType());
assertTrue("function name",funcName.equalsIgnoreCase(funcObj.getObjectName()));
assertEquals("db name",null,funcObj.getDbname());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ,List> io=getHivePrivilegeObjectInputs();
List inputs=io.getLeft();
assertEquals(1,inputs.size());
HivePrivilegeObject dbObj=inputs.get(0);
assertEquals("default",dbObj.getDbname().toLowerCase());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier outputs=getHivePrivilegeObjectInputs().getRight();
HivePrivilegeObject funcObj;
HivePrivilegeObject dbObj;
assertEquals("number of output object",2,outputs.size());
if (outputs.get(0).getType() == HivePrivilegeObjectType.FUNCTION) {
funcObj=outputs.get(0);
dbObj=outputs.get(1);
}
else {
funcObj=outputs.get(1);
dbObj=outputs.get(0);
}
assertEquals("input type",HivePrivilegeObjectType.FUNCTION,funcObj.getType());
assertTrue("function name",funcName.equalsIgnoreCase(funcObj.getObjectName()));
assertTrue("db name",dbName.equalsIgnoreCase(funcObj.getDbname()));
assertEquals("input type",HivePrivilegeObjectType.DATABASE,dbObj.getType());
assertTrue("db name",dbName.equalsIgnoreCase(dbObj.getDbname()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ,List> io=getHivePrivilegeObjectInputs();
List inputs=io.getLeft();
assertEquals(1,inputs.size());
HivePrivilegeObject tableObj=inputs.get(0);
assertEquals(1,tableObj.getColumns().size());
assertEquals("j",tableObj.getColumns().get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier inputs=getHivePrivilegeObjectInputs().getLeft();
checkSingleTableInput(inputs);
HivePrivilegeObject tableObj=inputs.get(0);
assertEquals("no of columns used",3,tableObj.getColumns().size());
assertEquals("Columns used",Arrays.asList("city","i","k"),getSortedList(tableObj.getColumns()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ,List> io=getHivePrivilegeObjectInputs();
List inputs=io.getLeft();
assertEquals(1,inputs.size());
HivePrivilegeObject dbObj=inputs.get(0);
assertEquals(dbName.toLowerCase(),dbObj.getDbname().toLowerCase());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier inputs=getHivePrivilegeObjectInputs().getLeft();
checkSingleTableInput(inputs);
HivePrivilegeObject tableObj=inputs.get(0);
assertEquals("no of columns used",5,tableObj.getColumns().size());
assertEquals("Columns used",Arrays.asList("city","date","i","j","k"),getSortedList(tableObj.getColumns()));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ,List> io=getHivePrivilegeObjectInputs();
List outputs=io.getRight();
HivePrivilegeObject tableObj=outputs.get(0);
LOG.debug("Got privilege object " + tableObj);
assertEquals("no of columns used",1,tableObj.getColumns().size());
assertEquals("Column used","i",tableObj.getColumns().get(0));
List inputs=io.getLeft();
assertEquals(1,inputs.size());
tableObj=inputs.get(0);
assertEquals(2,tableObj.getColumns().size());
assertEquals("j",tableObj.getColumns().get(0));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier ,List> io=getHivePrivilegeObjectInputs();
List outputs=io.getRight();
HivePrivilegeObject tableObj=outputs.get(0);
LOG.debug("Got privilege object " + tableObj);
assertEquals("no of columns used",2,tableObj.getColumns().size());
assertEquals("Columns used",Arrays.asList("i","l"),getSortedList(tableObj.getColumns()));
List inputs=io.getLeft();
assertEquals(1,inputs.size());
tableObj=inputs.get(0);
assertEquals(2,tableObj.getColumns().size());
assertEquals("Columns used",Arrays.asList("j","k"),getSortedList(tableObj.getColumns()));
}
Class: org.apache.hadoop.hive.ql.session.TestAddResource EqualityVerifier list=new LinkedList();
List addList=new LinkedList();
list.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
Mockito.when(ss.resolveAndDownload(query,false)).thenReturn(list);
addList.add(query);
ss.add_resources(t,addList);
Set dependencies=ss.list_resource(t,null);
LinkedList actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
Collections.sort(list);
Collections.sort(actual);
assertEquals(list,actual);
ss.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier list1=new LinkedList();
List list2=new LinkedList();
List list3=new LinkedList();
List addList=new LinkedList();
list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list3.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list3.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list3.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
Collections.sort(list1);
Collections.sort(list2);
Collections.sort(list3);
Mockito.when(ss.resolveAndDownload(query1,false)).thenReturn(list1);
Mockito.when(ss.resolveAndDownload(query2,false)).thenReturn(list2);
Mockito.when(ss.resolveAndDownload(query3,false)).thenReturn(list3);
addList.add(query1);
addList.add(query2);
addList.add(query3);
ss.add_resources(t,addList);
List deleteList=new LinkedList();
deleteList.add(list1.get(0).toString());
ss.delete_resources(t,deleteList);
Set dependencies=ss.list_resource(t,null);
LinkedList actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
List expected=union(list2,list3);
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected,actual);
actual.clear();
expected.clear();
deleteList.clear();
deleteList.add(list2.get(0).toString());
ss.delete_resources(t,deleteList);
dependencies=ss.list_resource(t,null);
actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
expected=new LinkedList(list3);
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected,actual);
actual.clear();
expected.clear();
deleteList.clear();
deleteList.add(list3.get(0).toString());
ss.delete_resources(t,deleteList);
dependencies=ss.list_resource(t,null);
assertEquals(dependencies.isEmpty(),true);
ss.close();
}
APIUtilityVerifier EqualityVerifier addList=new LinkedList();
List list1=new LinkedList();
List list2=new LinkedList();
list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
Mockito.when(ss.resolveAndDownload(query1,false)).thenReturn(list1);
Mockito.when(ss.resolveAndDownload(query2,false)).thenReturn(list2);
addList.add(query1);
addList.add(query2);
ss.add_resources(t,addList);
Set dependencies=ss.list_resource(t,null);
LinkedList actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
List expected=union(list1,list2);
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected,actual);
ss.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier list1=new LinkedList();
List list2=new LinkedList();
List addList=new LinkedList();
list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
Collections.sort(list1);
Collections.sort(list2);
Mockito.when(ss.resolveAndDownload(query1,false)).thenReturn(list1);
Mockito.when(ss.resolveAndDownload(query2,false)).thenReturn(list2);
addList.add(query1);
addList.add(query2);
ss.add_resources(t,addList);
List deleteList=new LinkedList();
deleteList.add(list1.get(0).toString());
ss.delete_resources(t,deleteList);
Set dependencies=ss.list_resource(t,null);
LinkedList actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
List expected=list2;
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected,actual);
deleteList.clear();
deleteList.add(list2.get(0).toString());
ss.delete_resources(t,deleteList);
dependencies=ss.list_resource(t,null);
assertEquals(dependencies.isEmpty(),true);
ss.close();
}
EqualityVerifier list=new LinkedList();
List addList=new LinkedList();
list.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
list.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
Collections.sort(list);
Mockito.when(ss.resolveAndDownload(query,false)).thenReturn(list);
for (int i=0; i < 10; i++) {
addList.add(query);
}
ss.add_resources(t,addList);
Set dependencies=ss.list_resource(t,null);
LinkedList actual=new LinkedList();
for ( String dependency : dependencies) {
actual.add(createURI(dependency));
}
Collections.sort(actual);
assertEquals(list,actual);
ss.close();
}
Class: org.apache.hadoop.hive.ql.session.TestSessionState UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier InternalCallVerifier EqualityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.txn.compactor.TestCleaner APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier paths=getDirectories(conf,t,p);
Assert.assertEquals(1,paths.size());
Assert.assertEquals("base_25",paths.get(0).getName());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
LockResponse res=txnHandler.lock(req);
AtomicBoolean looped=new AtomicBoolean();
looped.set(false);
startCleaner(looped);
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
LockComponent comp2=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"default");
comp2.setTablename("bblt");
List components2=new ArrayList(1);
components2.add(comp2);
LockRequest req2=new LockRequest(components,"me","localhost");
LockResponse res2=txnHandler.lock(req2);
txnHandler.unlock(new UnlockRequest(res.getLockid()));
looped.set(false);
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
stopThread();
Thread.currentThread().sleep(200);
rsp=txnHandler.showCompact(new ShowCompactRequest());
compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
LockResponse res=txnHandler.lock(req);
startCleaner();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
Assert.assertEquals("bblt",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MINOR,compacts.get(0).getType());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
LockResponse res=txnHandler.lock(req);
AtomicBoolean looped=new AtomicBoolean();
looped.set(false);
startCleaner(looped);
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
LockComponent comp2=new LockComponent(LockType.SHARED_READ,LockLevel.PARTITION,"default");
comp2.setTablename("bblt");
comp2.setPartitionname("ds=today");
List components2=new ArrayList(1);
components2.add(comp2);
LockRequest req2=new LockRequest(components,"me","localhost");
LockResponse res2=txnHandler.lock(req2);
txnHandler.unlock(new UnlockRequest(res.getLockid()));
looped.set(false);
while (!looped.get()) {
Thread.currentThread().sleep(100);
}
stopThread();
Thread.currentThread().sleep(200);
rsp=txnHandler.showCompact(new ShowCompactRequest());
compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
LockResponse res=txnHandler.lock(req);
startCleaner();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
Assert.assertEquals("bblp",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MINOR,compacts.get(0).getType());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier paths=getDirectories(conf,t,p);
Assert.assertEquals(1,paths.size());
Assert.assertEquals("base_25",paths.get(0).getName());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier paths=getDirectories(conf,t,null);
Assert.assertEquals(1,paths.size());
Assert.assertEquals("base_25",paths.get(0).getName());
}
APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier paths=getDirectories(conf,t,null);
Assert.assertEquals(2,paths.size());
boolean sawBase=false, sawDelta=false;
for ( Path p : paths) {
if (p.getName().equals("base_20")) sawBase=true;
else if (p.getName().equals(makeDeltaDirName(21,24))) sawDelta=true;
else Assert.fail("Unexpected file " + p.getName());
}
Assert.assertTrue(sawBase);
Assert.assertTrue(sawDelta);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier paths=getDirectories(conf,t,p);
Assert.assertEquals(2,paths.size());
boolean sawBase=false, sawDelta=false;
for ( Path path : paths) {
if (path.getName().equals("base_20")) sawBase=true;
else if (path.getName().equals(makeDeltaDirNameCompacted(21,24))) sawDelta=true;
else Assert.fail("Unexpected file " + path.getName());
}
Assert.assertTrue(sawBase);
Assert.assertTrue(sawDelta);
}
Class: org.apache.hadoop.hive.ql.txn.compactor.TestCompactor APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(ds string)"+ " CLUSTERED BY(a) INTO 2 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', "+ "'today'), (2, 'wilma', 'yesterday')",driver);
executeStatementOnDriver("update " + tblName + " set b = 'barney'",driver);
Initiator initiator=new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD,1);
initiator.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
stop.set(true);
initiator.init(stop,new AtomicBoolean());
initiator.run();
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(2,compacts.size());
SortedSet partNames=new TreeSet();
for (int i=0; i < compacts.size(); i++) {
Assert.assertEquals("default",compacts.get(i).getDbname());
Assert.assertEquals(tblName,compacts.get(i).getTablename());
Assert.assertEquals("initiated",compacts.get(i).getState());
partNames.add(compacts.get(i).getPartitionname());
}
List names=new ArrayList(partNames);
Assert.assertEquals("ds=today",names.get(0));
Assert.assertEquals("ds=yesterday",names.get(1));
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(ds string)"+ " CLUSTERED BY(a) INTO 2 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', "+ "'today'), (2, 'wilma', 'yesterday')",driver);
executeStatementOnDriver("update " + tblName + " set b = 'fred' where a = 1",driver);
executeStatementOnDriver("delete from " + tblName + " where b = 'fred'",driver);
Initiator initiator=new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD,2);
initiator.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
stop.set(true);
initiator.init(stop,new AtomicBoolean());
initiator.run();
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
SortedSet partNames=new TreeSet();
for (int i=0; i < compacts.size(); i++) {
Assert.assertEquals("default",compacts.get(i).getDbname());
Assert.assertEquals(tblName,compacts.get(i).getTablename());
Assert.assertEquals("initiated",compacts.get(i).getState());
partNames.add(compacts.get(i).getPartitionname());
}
List names=new ArrayList(partNames);
Assert.assertEquals("ds=today",names.get(0));
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(ds string)"+ " CLUSTERED BY(a) INTO 2 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', "+ "'today'), (2, 'wilma', 'yesterday')",driver);
Initiator initiator=new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD,0);
initiator.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
stop.set(true);
initiator.init(stop,new AtomicBoolean());
initiator.run();
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(2,compacts.size());
SortedSet partNames=new TreeSet();
for (int i=0; i < compacts.size(); i++) {
Assert.assertEquals("default",compacts.get(i).getDbname());
Assert.assertEquals(tblName,compacts.get(i).getTablename());
Assert.assertEquals("initiated",compacts.get(i).getState());
partNames.add(compacts.get(i).getPartitionname());
}
List names=new ArrayList(partNames);
Assert.assertEquals("ds=today",names.get(0));
Assert.assertEquals("ds=yesterday",names.get(1));
}
APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("drop table if exists " + tblNameStg,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(bkt INT)"+ " CLUSTERED BY(a) INTO 4 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("CREATE EXTERNAL TABLE " + tblNameStg + "(a INT, b STRING)"+ " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n'"+ " STORED AS TEXTFILE"+ " LOCATION '"+ stagingFolder.newFolder().toURI().getPath()+ "'"+ " TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("load data local inpath '" + BASIC_FILE_NAME + "' overwrite into table "+ tblNameStg,driver);
execSelectAndDumpData("select * from " + tblNameStg,driver,"Dumping data for " + tblNameStg + " after load:");
executeStatementOnDriver("FROM " + tblNameStg + " INSERT INTO TABLE "+ tblName+ " PARTITION(bkt=0) "+ "SELECT a, b where a < 2",driver);
executeStatementOnDriver("FROM " + tblNameStg + " INSERT INTO TABLE "+ tblName+ " PARTITION(bkt=1) "+ "SELECT a, b where a >= 2",driver);
execSelectAndDumpData("select * from " + tblName,driver,"Dumping data for " + tblName + " after load:");
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
CompactionInfo ci=new CompactionInfo("default",tblName,"bkt=0",CompactionType.MAJOR);
LOG.debug("List of stats columns before analyze Part1: " + txnHandler.findColumnsWithStats(ci));
Worker.StatsUpdater su=Worker.StatsUpdater.init(ci,colNames,conf,System.getProperty("user.name"));
su.gatherStats();
LOG.debug("List of stats columns after analyze Part1: " + txnHandler.findColumnsWithStats(ci));
CompactionInfo ciPart2=new CompactionInfo("default",tblName,"bkt=1",CompactionType.MAJOR);
LOG.debug("List of stats columns before analyze Part2: " + txnHandler.findColumnsWithStats(ci));
su=Worker.StatsUpdater.init(ciPart2,colNames,conf,System.getProperty("user.name"));
su.gatherStats();
LOG.debug("List of stats columns after analyze Part2: " + txnHandler.findColumnsWithStats(ci));
Map> stats=msClient.getPartitionColumnStatistics(ci.dbname,ci.tableName,Arrays.asList(ci.partName),colNames);
List colStats=stats.get(ci.partName);
Assert.assertNotNull("No stats found for partition " + ci.partName,colStats);
Assert.assertEquals("Expected column 'a' at index 0","a",colStats.get(0).getColName());
Assert.assertEquals("Expected column 'b' at index 1","b",colStats.get(1).getColName());
LongColumnStatsData colAStats=colStats.get(0).getStatsData().getLongStats();
Assert.assertEquals("lowValue a",1,colAStats.getLowValue());
Assert.assertEquals("highValue a",1,colAStats.getHighValue());
Assert.assertEquals("numNulls a",0,colAStats.getNumNulls());
Assert.assertEquals("numNdv a",1,colAStats.getNumDVs());
StringColumnStatsData colBStats=colStats.get(1).getStatsData().getStringStats();
Assert.assertEquals("maxColLen b",3,colBStats.getMaxColLen());
Assert.assertEquals("avgColLen b",3.0,colBStats.getAvgColLen(),0.01);
Assert.assertEquals("numNulls b",0,colBStats.getNumNulls());
Assert.assertEquals("nunDVs",2,colBStats.getNumDVs());
stats=msClient.getPartitionColumnStatistics(ciPart2.dbname,ciPart2.tableName,Arrays.asList(ciPart2.partName),colNames);
colStats=stats.get(ciPart2.partName);
LongColumnStatsData colAStatsPart2=colStats.get(0).getStatsData().getLongStats();
StringColumnStatsData colBStatsPart2=colStats.get(1).getStatsData().getStringStats();
HiveEndPoint endPt=new HiveEndPoint(null,ci.dbname,ci.tableName,Arrays.asList("0"));
DelimitedInputWriter writer=new DelimitedInputWriter(new String[]{"a","b"},",",endPt);
StreamingConnection connection=endPt.newConnection(true);
TransactionBatch txnBatch=connection.fetchTransactionBatch(2,writer);
txnBatch.beginNextTransaction();
Assert.assertEquals(TransactionBatch.TxnState.OPEN,txnBatch.getCurrentTransactionState());
txnBatch.write("50,Kiev".getBytes());
txnBatch.write("51,St. Petersburg".getBytes());
txnBatch.write("44,Boston".getBytes());
txnBatch.commit();
txnBatch.beginNextTransaction();
txnBatch.write("52,Tel Aviv".getBytes());
txnBatch.write("53,Atlantis".getBytes());
txnBatch.write("53,Boston".getBytes());
txnBatch.commit();
txnBatch.close();
connection.close();
execSelectAndDumpData("select * from " + ci.getFullTableName(),driver,ci.getFullTableName());
CompactionRequest rqst=new CompactionRequest(ci.dbname,ci.tableName,CompactionType.MAJOR);
rqst.setPartitionname(ci.partName);
txnHandler.compact(rqst);
Worker t=new Worker();
t.setThreadId((int)t.getId());
t.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
AtomicBoolean looped=new AtomicBoolean();
stop.set(true);
t.init(stop,looped);
t.run();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
if (1 != compacts.size()) {
Assert.fail("Expecting 1 file and found " + compacts.size() + " files "+ compacts.toString());
}
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
stats=msClient.getPartitionColumnStatistics(ci.dbname,ci.tableName,Arrays.asList(ci.partName),colNames);
colStats=stats.get(ci.partName);
Assert.assertNotNull("No stats found for partition " + ci.partName,colStats);
Assert.assertEquals("Expected column 'a' at index 0","a",colStats.get(0).getColName());
Assert.assertEquals("Expected column 'b' at index 1","b",colStats.get(1).getColName());
colAStats=colStats.get(0).getStatsData().getLongStats();
Assert.assertEquals("lowValue a",1,colAStats.getLowValue());
Assert.assertEquals("highValue a",53,colAStats.getHighValue());
Assert.assertEquals("numNulls a",0,colAStats.getNumNulls());
Assert.assertEquals("numNdv a",6,colAStats.getNumDVs());
colBStats=colStats.get(1).getStatsData().getStringStats();
Assert.assertEquals("maxColLen b",14,colBStats.getMaxColLen());
Assert.assertEquals("avgColLen b",(long)6.1111111111,(long)colBStats.getAvgColLen());
Assert.assertEquals("numNulls b",0,colBStats.getNumNulls());
Assert.assertEquals("nunDVs",10,colBStats.getNumDVs());
stats=msClient.getPartitionColumnStatistics(ciPart2.dbname,ciPart2.tableName,Arrays.asList(ciPart2.partName),colNames);
colStats=stats.get(ciPart2.partName);
Assert.assertEquals("Expected stats for " + ciPart2.partName + " to stay the same",colAStatsPart2,colStats.get(0).getStatsData().getLongStats());
Assert.assertEquals("Expected stats for " + ciPart2.partName + " to stay the same",colBStatsPart2,colStats.get(1).getStatsData().getStringStats());
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(ds string)"+ " CLUSTERED BY(a) INTO 2 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', "+ "'today'), (2, 'wilma', 'yesterday')",driver);
executeStatementOnDriver("ALTER TABLE " + tblName + " ADD COLUMNS(c int)",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
ArrayList valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(2,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tfred\tNULL\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\twilma\tNULL\tyesterday",valuesReadFromHiveDriver.get(1));
executeStatementOnDriver("insert into " + tblName + " partition (ds) values "+ "(3, 'mark', 1900, 'soon'), (4, 'douglas', 1901, 'last_century'), "+ "(5, 'doc', 1902, 'yesterday')",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(5,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tfred\tNULL\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\twilma\tNULL\tyesterday",valuesReadFromHiveDriver.get(1));
Assert.assertEquals("3\tmark\t1900\tsoon",valuesReadFromHiveDriver.get(2));
Assert.assertEquals("4\tdouglas\t1901\tlast_century",valuesReadFromHiveDriver.get(3));
Assert.assertEquals("5\tdoc\t1902\tyesterday",valuesReadFromHiveDriver.get(4));
Initiator initiator=new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD,0);
initiator.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
stop.set(true);
initiator.init(stop,new AtomicBoolean());
initiator.run();
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(4,compacts.size());
SortedSet partNames=new TreeSet();
for (int i=0; i < compacts.size(); i++) {
Assert.assertEquals("default",compacts.get(i).getDbname());
Assert.assertEquals(tblName,compacts.get(i).getTablename());
Assert.assertEquals("initiated",compacts.get(i).getState());
partNames.add(compacts.get(i).getPartitionname());
}
List names=new ArrayList(partNames);
Assert.assertEquals("ds=last_century",names.get(0));
Assert.assertEquals("ds=soon",names.get(1));
Assert.assertEquals("ds=today",names.get(2));
Assert.assertEquals("ds=yesterday",names.get(3));
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(5,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tfred\tNULL\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\twilma\tNULL\tyesterday",valuesReadFromHiveDriver.get(1));
Assert.assertEquals("3\tmark\t1900\tsoon",valuesReadFromHiveDriver.get(2));
Assert.assertEquals("4\tdouglas\t1901\tlast_century",valuesReadFromHiveDriver.get(3));
Assert.assertEquals("5\tdoc\t1902\tyesterday",valuesReadFromHiveDriver.get(4));
}
APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier colNames=Arrays.asList("a","b");
String columnNamesProperty="a,b";
String columnTypesProperty="int:string";
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " CLUSTERED BY(a) INTO 1 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true') ",driver);
HiveEndPoint endPt=new HiveEndPoint(null,dbName,tblName,null);
DelimitedInputWriter writer=new DelimitedInputWriter(new String[]{"a","b"},",",endPt);
StreamingConnection connection=endPt.newConnection(false);
try {
for (int i=0; i < 2; i++) {
writeBatch(connection,writer,false);
}
writeBatch(connection,writer,true);
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
txnHandler.compact(new CompactionRequest(dbName,tblName,CompactionType.MAJOR));
Worker t=new Worker();
t.setThreadId((int)t.getId());
t.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean(true);
AtomicBoolean looped=new AtomicBoolean();
t.init(stop,looped);
t.run();
IMetaStoreClient msClient=new HiveMetaStoreClient(conf);
Table table=msClient.getTable(dbName,tblName);
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(table.getSd().getLocation()),AcidUtils.baseFileFilter);
if (1 != stat.length) {
Assert.fail("Expecting 1 file \"base_0000004\" and found " + stat.length + " files "+ Arrays.toString(stat));
}
String name=stat[0].getPath().getName();
Assert.assertEquals(name,"base_0000004");
checkExpectedTxnsPresent(stat[0].getPath(),null,columnNamesProperty,columnTypesProperty,0,1L,4L);
}
finally {
connection.close();
}
}
APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier colNames=Arrays.asList("a","b");
executeStatementOnDriver("drop table if exists " + tblName,driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) "+ " PARTITIONED BY(ds string)"+ " CLUSTERED BY(a) INTO 2 BUCKETS"+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')",driver);
executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', "+ "'today'), (2, 'wilma', 'yesterday')",driver);
executeStatementOnDriver("update " + tblName + " set b = 'barney'",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
ArrayList valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(2,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tbarney\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\tbarney\tyesterday",valuesReadFromHiveDriver.get(1));
executeStatementOnDriver("ALTER TABLE " + tblName + " ADD COLUMNS(c int)",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(2,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tbarney\tNULL\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\tbarney\tNULL\tyesterday",valuesReadFromHiveDriver.get(1));
executeStatementOnDriver("insert into " + tblName + " partition (ds) values "+ "(3, 'mark', 1900, 'soon'), (4, 'douglas', 1901, 'last_century'), "+ "(5, 'doc', 1902, 'yesterday')",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(5,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tbarney\tNULL\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\tbarney\tNULL\tyesterday",valuesReadFromHiveDriver.get(1));
Assert.assertEquals("3\tmark\t1900\tsoon",valuesReadFromHiveDriver.get(2));
Assert.assertEquals("4\tdouglas\t1901\tlast_century",valuesReadFromHiveDriver.get(3));
Assert.assertEquals("5\tdoc\t1902\tyesterday",valuesReadFromHiveDriver.get(4));
executeStatementOnDriver("update " + tblName + " set c = 2000",driver);
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(5,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tbarney\t2000\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\tbarney\t2000\tyesterday",valuesReadFromHiveDriver.get(1));
Assert.assertEquals("3\tmark\t2000\tsoon",valuesReadFromHiveDriver.get(2));
Assert.assertEquals("4\tdouglas\t2000\tlast_century",valuesReadFromHiveDriver.get(3));
Assert.assertEquals("5\tdoc\t2000\tyesterday",valuesReadFromHiveDriver.get(4));
Initiator initiator=new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD,1);
initiator.setHiveConf(conf);
AtomicBoolean stop=new AtomicBoolean();
stop.set(true);
initiator.init(stop,new AtomicBoolean());
initiator.run();
TxnStore txnHandler=TxnUtils.getTxnStore(conf);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(4,compacts.size());
SortedSet partNames=new TreeSet();
for (int i=0; i < compacts.size(); i++) {
Assert.assertEquals("default",compacts.get(i).getDbname());
Assert.assertEquals(tblName,compacts.get(i).getTablename());
Assert.assertEquals("initiated",compacts.get(i).getState());
partNames.add(compacts.get(i).getPartitionname());
}
List names=new ArrayList(partNames);
Assert.assertEquals("ds=last_century",names.get(0));
Assert.assertEquals("ds=soon",names.get(1));
Assert.assertEquals("ds=today",names.get(2));
Assert.assertEquals("ds=yesterday",names.get(3));
executeStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a",driver);
valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
Assert.assertEquals(5,valuesReadFromHiveDriver.size());
Assert.assertEquals("1\tbarney\t2000\ttoday",valuesReadFromHiveDriver.get(0));
Assert.assertEquals("2\tbarney\t2000\tyesterday",valuesReadFromHiveDriver.get(1));
Assert.assertEquals("3\tmark\t2000\tsoon",valuesReadFromHiveDriver.get(2));
Assert.assertEquals("4\tdouglas\t2000\tlast_century",valuesReadFromHiveDriver.get(3));
Assert.assertEquals("5\tdoc\t2000\tyesterday",valuesReadFromHiveDriver.get(4));
}
Class: org.apache.hadoop.hive.ql.txn.compactor.TestInitiator InternalCallVerifier EqualityVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("cttmd",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MINOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("ednb",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
ms.dropTable("default","dt");
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(0,compacts.size());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(0,compacts.size());
}
EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
for (int i=0; i < TxnStore.TIMED_OUT_TXN_ABORT_BATCH_SIZE + 50; i++) {
txnid=openTxn();
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
GetOpenTxnsResponse openTxns=txnHandler.getOpenTxns();
Assert.assertEquals(TxnStore.TIMED_OUT_TXN_ABORT_BATCH_SIZE + 50 + 1,openTxns.getOpen_txnsSize());
startInitiator();
openTxns=txnHandler.getOpenTxns();
Assert.assertEquals(1,openTxns.getOpen_txnsSize());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("cmomwbv",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
ms.dropPartition("default","dp",Collections.singletonList("today"),true);
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(0,compacts.size());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
CompactionRequest rqst=new CompactionRequest("default","ncwcas",CompactionType.MAJOR);
txnHandler.compact(rqst);
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("ncwcas",compacts.get(0).getTablename());
startInitiator();
rsp=txnHandler.showCompact(new ShowCompactRequest());
compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("ncwcas",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0,rsp.getCompactsSize());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("cphdp",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0,rsp.getCompactsSize());
}
EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0,rsp.getCompactsSize());
}
BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(2,compacts.size());
boolean sawInitiated=false;
for ( ShowCompactResponseElement c : compacts) {
if (c.getState().equals("working")) {
Assert.assertEquals("nosuchhost-193892",c.getWorkerid());
}
else if (c.getState().equals("initiated")) {
sawInitiated=true;
}
else {
Assert.fail("Unexpected state");
}
}
Assert.assertTrue(sawInitiated);
}
EqualityVerifier parameters=new HashMap(1);
parameters.put("NO_AUTO_COMPACTION","true");
Table t=newTable("default","ncwncs",false,parameters);
HiveConf.setIntVar(conf,HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD,10);
for (int i=0; i < 11; i++) {
long txnid=openTxn();
LockComponent comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.TABLE,"default");
comp.setTablename("ncwncs");
List components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0,rsp.getCompactsSize());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
txnid=openTxn();
comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.PARTITION,"default");
comp.setTablename("ttospgocr");
comp.setPartitionname("ds=today");
components=new ArrayList(1);
components.add(comp);
req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("ttospgocr",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("cptmd",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MINOR,compacts.get(0).getType());
}
EqualityVerifier parameters=new HashMap(1);
parameters.put("no_auto_compaction","true");
Table t=newTable("default","ncwncs",false,parameters);
HiveConf.setIntVar(conf,HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD,10);
for (int i=0; i < 11; i++) {
long txnid=openTxn();
LockComponent comp=new LockComponent(LockType.SHARED_WRITE,LockLevel.TABLE,"default");
comp.setTablename("ncwncs");
List components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
Assert.assertEquals(0,rsp.getCompactsSize());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("mcottma",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.commitTxn(new CommitTxnRequest(txnid));
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("cthdp",compacts.get(0).getTablename());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
InternalCallVerifier EqualityVerifier components=new ArrayList(1);
components.add(comp);
LockRequest req=new LockRequest(components,"me","localhost");
req.setTxnid(txnid);
LockResponse res=txnHandler.lock(req);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
}
startInitiator();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("initiated",compacts.get(0).getState());
Assert.assertEquals("mcoptma",compacts.get(0).getTablename());
Assert.assertEquals("ds=today",compacts.get(0).getPartitionname());
Assert.assertEquals(CompactionType.MAJOR,compacts.get(0).getType());
}
Class: org.apache.hadoop.hive.ql.txn.compactor.TestWorker BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
boolean sawNewDelta=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(21,24))) {
sawNewDelta=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(208L,buckets[0].getLen());
Assert.assertEquals(208L,buckets[1].getLen());
}
else {
LOG.debug("This is not the delta file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewDelta);
}
InternalCallVerifier EqualityVerifier (Arrays.asList(24L,25L)));
CompactionRequest rqst=new CompactionRequest("default","mtwb",CompactionType.MINOR);
txnHandler.compact(rqst);
startWorker();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(5,stat.length);
Arrays.sort(stat);
Assert.assertEquals("base_20",stat[0].getPath().getName());
Assert.assertEquals(makeDeltaDirName(21,22),stat[1].getPath().getName());
Assert.assertEquals(makeDeltaDirNameCompacted(21,27),stat[2].getPath().getName());
Assert.assertEquals(makeDeltaDirName(23,25),stat[3].getPath().getName());
Assert.assertEquals(makeDeltaDirName(26,27),stat[4].getPath().getName());
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(p.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
boolean sawNewBase=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals("base_0000024")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(624L,buckets[0].getLen());
Assert.assertEquals(624L,buckets[1].getLen());
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewBase);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(compacts.get(0).getState()));
}
InternalCallVerifier EqualityVerifier (Arrays.asList(23L)),null);
CompactionRequest rqst=new CompactionRequest("default","mtwb",CompactionType.MAJOR);
txnHandler.compact(rqst);
startWorker();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(5,stat.length);
Arrays.sort(stat);
Assert.assertEquals("base_0000022",stat[0].getPath().getName());
Assert.assertEquals("base_20",stat[1].getPath().getName());
Assert.assertEquals(makeDeltaDirName(21,22),stat[2].getPath().getName());
Assert.assertEquals(makeDeltaDirName(23,25),stat[3].getPath().getName());
Assert.assertEquals(makeDeltaDirName(26,27),stat[4].getPath().getName());
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(p.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
boolean sawNewDelta=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(21,24))) {
sawNewDelta=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(208L,buckets[0].getLen());
Assert.assertEquals(208L,buckets[1].getLen());
}
else {
LOG.debug("This is not the delta file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewDelta);
}
IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
boolean sawNewDelta=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(21,24))) {
sawNewDelta=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewDelta);
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier files=new ArrayList(1);
files.add(file);
Path[] deltas=new Path[2];
deltas[0]=new Path(delta1);
deltas[1]=new Path(delta2);
CompactorMR.CompactorInputSplit split=new CompactorMR.CompactorInputSplit(conf,3,files,null,deltas);
ByteArrayOutputStream buf=new ByteArrayOutputStream();
DataOutput out=new DataOutputStream(buf);
split.write(out);
split=new CompactorMR.CompactorInputSplit();
DataInput in=new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
split.readFields(in);
Assert.assertEquals(3,split.getBucket());
Assert.assertNull(split.getBaseDir());
deltas=split.getDeltaDirs();
Assert.assertEquals(2,deltas.length);
Assert.assertEquals(delta1,deltas[0].toString());
Assert.assertEquals(delta2,deltas[1].toString());
}
InternalCallVerifier EqualityVerifier (Arrays.asList(23L)),null);
CompactionRequest rqst=new CompactionRequest("default","mtwb",CompactionType.MINOR);
txnHandler.compact(rqst);
startWorker();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
Arrays.sort(stat);
Assert.assertEquals("base_20",stat[0].getPath().getName());
Assert.assertEquals(makeDeltaDirNameCompacted(21,22),stat[1].getPath().getName());
Assert.assertEquals(makeDeltaDirName(23,25),stat[2].getPath().getName());
Assert.assertEquals(makeDeltaDirName(26,27),stat[3].getPath().getName());
}
IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(3,stat.length);
boolean sawNewDelta=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals(makeDeltaDirNameCompacted(1,4))) {
sawNewDelta=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(208L,buckets[0].getLen());
Assert.assertEquals(208L,buckets[1].getLen());
}
else {
LOG.debug("This is not the delta file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewDelta);
}
InternalCallVerifier EqualityVerifier (Arrays.asList(24L,25L)));
CompactionRequest rqst=new CompactionRequest("default","mtwb",CompactionType.MAJOR);
txnHandler.compact(rqst);
startWorker();
ShowCompactResponse rsp=txnHandler.showCompact(new ShowCompactRequest());
List compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(5,stat.length);
Arrays.sort(stat);
Assert.assertEquals("base_0000027",stat[0].getPath().getName());
Assert.assertEquals("base_20",stat[1].getPath().getName());
Assert.assertEquals(makeDeltaDirName(21,22),stat[2].getPath().getName());
Assert.assertEquals(makeDeltaDirName(23,25),stat[3].getPath().getName());
Assert.assertEquals(makeDeltaDirName(26,27),stat[4].getPath().getName());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier sortCols=new ArrayList(1);
sortCols.add(new Order("b",1));
Table t=newTable("default","sp",true,new HashMap(),sortCols,false);
Partition p=newPartition(t,"today",sortCols);
addBaseFile(t,p,20L,20);
addDeltaFile(t,p,21L,22L,2);
addDeltaFile(t,p,23L,24L,2);
addDeltaFile(t,p,21L,24L,4);
burnThroughTransactions(25);
CompactionRequest rqst=new CompactionRequest("default","sp",CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
startWorker();
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(p.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(p.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
boolean sawNewBase=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals("base_0000026")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(("bucket_00000".equals(buckets[0].getPath().getName()) && 104L == buckets[0].getLen() && "bucket_00001".equals(buckets[1].getPath().getName()) && 676L == buckets[1].getLen()) || ("bucket_00000".equals(buckets[1].getPath().getName()) && 104L == buckets[1].getLen() && "bucket_00001".equals(buckets[0].getPath().getName()) && 676L == buckets[0].getLen()));
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewBase);
}
IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
boolean sawNewBase=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals("base_0000024")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(624L,buckets[0].getLen());
Assert.assertEquals(624L,buckets[1].getLen());
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewBase);
}
BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ());
String s=m.toString();
Assert.assertEquals("0:",s);
m=new CompactorMR.StringableMap(s);
Assert.assertEquals(0,m.size());
Map base=new HashMap();
base.put("mary","poppins");
base.put("bert",null);
base.put(null,"banks");
m=new CompactorMR.StringableMap(base);
s=m.toString();
m=new CompactorMR.StringableMap(s);
Assert.assertEquals(3,m.size());
Map saw=new HashMap(3);
saw.put("mary",false);
saw.put("bert",false);
saw.put(null,false);
for ( Map.Entry e : m.entrySet()) {
saw.put(e.getKey(),true);
if ("mary".equals(e.getKey())) Assert.assertEquals("poppins",e.getValue());
else if ("bert".equals(e.getKey())) Assert.assertNull(e.getValue());
else if (null == e.getKey()) Assert.assertEquals("banks",e.getValue());
else Assert.fail("Unexpected value " + e.getKey());
}
Assert.assertEquals(3,saw.size());
Assert.assertTrue(saw.get("mary"));
Assert.assertTrue(saw.get("bert"));
Assert.assertTrue(saw.get(null));
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier sortCols=new ArrayList(1);
sortCols.add(new Order("b",1));
Table t=newTable("default","st",false,new HashMap(),sortCols,false);
addBaseFile(t,null,20L,20);
addDeltaFile(t,null,21L,22L,2);
addDeltaFile(t,null,23L,24L,2);
addDeltaFile(t,null,21L,24L,4);
burnThroughTransactions(25);
CompactionRequest rqst=new CompactionRequest("default","st",CompactionType.MINOR);
txnHandler.compact(rqst);
startWorker();
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(4,stat.length);
}
IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
boolean sawNewBase=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals("base_0000024")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(624L,buckets[0].getLen());
Assert.assertEquals(624L,buckets[1].getLen());
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewBase);
}
InternalCallVerifier EqualityVerifier files=new ArrayList(1);
files.add(file);
Path[] deltas=new Path[2];
deltas[0]=new Path(delta1);
deltas[1]=new Path(delta2);
CompactorMR.CompactorInputSplit split=new CompactorMR.CompactorInputSplit(conf,3,files,new Path(basename),deltas);
Assert.assertEquals(520L,split.getLength());
String[] locations=split.getLocations();
Assert.assertEquals(1,locations.length);
Assert.assertEquals("localhost",locations[0]);
ByteArrayOutputStream buf=new ByteArrayOutputStream();
DataOutput out=new DataOutputStream(buf);
split.write(out);
split=new CompactorMR.CompactorInputSplit();
DataInput in=new DataInputStream(new ByteArrayInputStream(buf.toByteArray()));
split.readFields(in);
Assert.assertEquals(3,split.getBucket());
Assert.assertEquals(basename,split.getBaseDir().toString());
deltas=split.getDeltaDirs();
Assert.assertEquals(2,deltas.length);
Assert.assertEquals(delta1,deltas[0].toString());
Assert.assertEquals(delta2,deltas[1].toString());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier compacts=rsp.getCompacts();
Assert.assertEquals(1,compacts.size());
Assert.assertEquals("ready for cleaning",compacts.get(0).getState());
FileSystem fs=FileSystem.get(conf);
FileStatus[] stat=fs.listStatus(new Path(t.getSd().getLocation()));
Assert.assertEquals(3,stat.length);
boolean sawNewBase=false;
for (int i=0; i < stat.length; i++) {
if (stat[i].getPath().getName().equals("base_0000004")) {
sawNewBase=true;
FileStatus[] buckets=fs.listStatus(stat[i].getPath());
Assert.assertEquals(2,buckets.length);
Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
Assert.assertEquals(104L,buckets[0].getLen());
Assert.assertEquals(104L,buckets[1].getLen());
}
else {
LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
}
}
Assert.assertTrue(sawNewBase);
}
Class: org.apache.hadoop.hive.ql.udf.TestBlockedUdf EqualityVerifier EqualityVerifier funcNames=FunctionRegistry.getFunctionNames();
funcNames.remove("reflect");
FunctionRegistry.setupPermissionsForBuiltinUDFs(funcNames.toString(),"");
assertEquals("substr",FunctionRegistry.getFunctionInfo("substr").getDisplayName());
}
EqualityVerifier ExceptionVerifier HybridVerifier EqualityVerifier ExceptionVerifier HybridVerifier EqualityVerifier ExceptionVerifier HybridVerifier funcNames=FunctionRegistry.getFunctionNames();
funcNames.remove("reflect");
FunctionRegistry.setupPermissionsForBuiltinUDFs(funcNames.toString(),"");
assertEquals("reflect",FunctionRegistry.getFunctionInfo("reflect").getDisplayName());
}
EqualityVerifier ExceptionVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.udf.TestToInteger InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.udf.TestUDFMath InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.TestUDFSign InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFBRound APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFCeil InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFFloor InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFMacro BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPDivide InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPMinus InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPMod InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPMultiply InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPNegative InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPPlus InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFOPPositive InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFPosMod InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFPower InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFPrintf InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFRound InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.generic.TestGenericUDFUtils EqualityVerifier Class: org.apache.hadoop.hive.ql.udf.xml.TestReusableStringReader APIUtilityVerifier EqualityVerifier APIUtilityVerifier BranchVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.ql.udf.xml.TestUDFXPathUtil BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier b1 b2 b3 c1 c2 true false b3 c1 c2 true false b3 c1 c2 true false b3 c1 c2 true false b3 c1 c2 true FALSE b3 c1 c2 true false b3 c1 -77 true FALSE b3 c1 c2 true false b3 c1 -77 true false b3 c1 -77 Class: org.apache.hadoop.hive.serde2.TestColumnProjectionUtils APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier columnIds=new ArrayList();
List actual;
assertEquals(Collections.EMPTY_LIST,ColumnProjectionUtils.getReadColumnIDs(conf));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
assertEquals(Collections.EMPTY_LIST,actual);
ColumnProjectionUtils.appendReadColumns(conf,columnIds);
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
assertEquals(Collections.EMPTY_LIST,actual);
assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
columnIds.add(1);
ColumnProjectionUtils.appendReadColumns(conf,Collections.singletonList(1));
assertEquals(columnIds,ColumnProjectionUtils.getReadColumnIDs(conf));
columnIds.add(2);
ColumnProjectionUtils.appendReadColumns(conf,Collections.singletonList(2));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
Collections.sort(actual);
assertEquals(columnIds,actual);
columnIds.add(3);
ColumnProjectionUtils.appendReadColumns(conf,Collections.singletonList(3));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
Collections.sort(actual);
assertEquals(columnIds,actual);
assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
}
APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier columnIds=new ArrayList();
List actual;
assertEquals(Collections.EMPTY_LIST,ColumnProjectionUtils.getReadColumnIDs(conf));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
assertEquals(Collections.EMPTY_LIST,actual);
ColumnProjectionUtils.setReadColumnIDs(conf,columnIds);
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
assertEquals(Collections.EMPTY_LIST,actual);
columnIds.add(1);
ColumnProjectionUtils.setReadColumnIDs(conf,Collections.singletonList(1));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
Collections.sort(actual);
assertEquals(columnIds,actual);
columnIds.add(2);
ColumnProjectionUtils.appendReadColumnIDs(conf,Collections.singletonList(2));
actual=ColumnProjectionUtils.getReadColumnIDs(conf);
Collections.sort(actual);
assertEquals(columnIds,actual);
assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
ColumnProjectionUtils.setFullyReadColumns(conf);
assertTrue(ColumnProjectionUtils.isReadAllColumns(conf));
}
BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.TestOpenCSVSerde InternalCallVerifier EqualityVerifier row=(List)csv.deserialize(in);
assertEquals("hello",row.get(0));
assertEquals("yes, okay",row.get(1));
assertEquals("1",row.get(2));
}
InternalCallVerifier EqualityVerifier row=(List)csv.deserialize(in);
assertEquals("hello",row.get(0));
assertEquals("yes\tokay",row.get(1));
assertEquals("1",row.get(2));
}
InternalCallVerifier EqualityVerifier row=(List)csv.deserialize(in);
assertEquals("hello",row.get(0));
assertEquals("yes'okay",row.get(1));
assertEquals("1",row.get(2));
}
Class: org.apache.hadoop.hive.serde2.avro.TestAvroDeserializer APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier m=new Hashtable();
m.put("one",1l);
m.put("two",2l);
m.put("three",3l);
record.put("aMap",m);
assertTrue(GENERIC_DATA.validate(s,record));
System.out.println("record = " + record);
AvroGenericRecordWritable garw=Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig=new AvroObjectInspectorGenerator(s);
AvroDeserializer de=new AvroDeserializer();
ArrayList row=(ArrayList)de.deserialize(aoig.getColumnNames(),aoig.getColumnTypes(),garw,s);
assertEquals(1,row.size());
Object theMapObject=row.get(0);
assertTrue(theMapObject instanceof Map);
Map theMap=(Map)theMapObject;
assertEquals(1l,theMap.get("one"));
assertEquals(2l,theMap.get("two"));
assertEquals(3l,theMap.get("three"));
StandardStructObjectInspector oi=(StandardStructObjectInspector)aoig.getObjectInspector();
List z=oi.getStructFieldsDataAsList(row);
assertEquals(1,z.size());
StructField fieldRef=oi.getStructFieldRef("amap");
Map theMap2=(Map)oi.getStructFieldData(row,fieldRef);
assertEquals(1l,theMap2.get("one"));
assertEquals(2l,theMap2.get("two"));
assertEquals(3l,theMap2.get("three"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier list=new ArrayList();
list.add("Eccleston");
list.add("Tennant");
list.add("Smith");
record.put("anArray",list);
assertTrue(GENERIC_DATA.validate(s,record));
System.out.println("Array-backed record = " + record);
AvroGenericRecordWritable garw=Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig=new AvroObjectInspectorGenerator(s);
AvroDeserializer de=new AvroDeserializer();
ArrayList row=(ArrayList)de.deserialize(aoig.getColumnNames(),aoig.getColumnTypes(),garw,s);
assertEquals(1,row.size());
Object theArrayObject=row.get(0);
assertTrue(theArrayObject instanceof List);
List theList=(List)theArrayObject;
assertEquals("Eccleston",theList.get(0));
assertEquals("Tennant",theList.get(1));
assertEquals("Smith",theList.get(2));
StandardStructObjectInspector oi=(StandardStructObjectInspector)aoig.getObjectInspector();
StructField fieldRefToArray=oi.getStructFieldRef("anArray");
Object anArrayData=oi.getStructFieldData(row,fieldRefToArray);
StandardListObjectInspector anArrayOI=(StandardListObjectInspector)fieldRefToArray.getFieldObjectInspector();
assertEquals(3,anArrayOI.getListLength(anArrayData));
JavaStringObjectInspector elementOI=(JavaStringObjectInspector)anArrayOI.getListElementObjectInspector();
Object firstElement=anArrayOI.getListElement(anArrayData,0);
assertEquals("Eccleston",elementOI.getPrimitiveJavaObject(firstElement));
assertTrue(firstElement instanceof String);
Object secondElement=anArrayOI.getListElement(anArrayData,1);
assertEquals("Tennant",elementOI.getPrimitiveJavaObject(secondElement));
assertTrue(secondElement instanceof String);
Object thirdElement=anArrayOI.getListElement(anArrayData,2);
assertEquals("Smith",elementOI.getPrimitiveJavaObject(thirdElement));
assertTrue(thirdElement instanceof String);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier m=new HashMap();
m.put("one",1l);
m.put("two",2l);
m.put("three",3l);
m.put("mu",null);
record.put("aMap",m);
assertTrue(GENERIC_DATA.validate(s,record));
System.out.println("record = " + record);
AvroGenericRecordWritable garw=Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig=new AvroObjectInspectorGenerator(s);
AvroDeserializer de=new AvroDeserializer();
ArrayList row=(ArrayList)de.deserialize(aoig.getColumnNames(),aoig.getColumnTypes(),garw,s);
assertEquals(1,row.size());
Object theMapObject=row.get(0);
assertTrue(theMapObject instanceof Map);
Map theMap=(Map)theMapObject;
assertEquals(1l,theMap.get("one"));
assertEquals(2l,theMap.get("two"));
assertEquals(3l,theMap.get("three"));
assertTrue(theMap.containsKey("mu"));
assertEquals(null,theMap.get("mu"));
StandardStructObjectInspector oi=(StandardStructObjectInspector)aoig.getObjectInspector();
List z=oi.getStructFieldsDataAsList(row);
assertEquals(1,z.size());
StructField fieldRef=oi.getStructFieldRef("amap");
Map theMap2=(Map)oi.getStructFieldData(row,fieldRef);
assertEquals(1l,theMap2.get("one"));
assertEquals(2l,theMap2.get("two"));
assertEquals(3l,theMap2.get("three"));
assertTrue(theMap2.containsKey("mu"));
assertEquals(null,theMap2.get("mu"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier m=new Hashtable();
m.put("one",1l);
m.put("two",2l);
m.put("three",3l);
record.put("aMap",m);
assertTrue(GENERIC_DATA.validate(s,record));
System.out.println("record = " + record);
AvroGenericRecordWritable garw=Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig=new AvroObjectInspectorGenerator(s);
AvroDeserializer de=new AvroDeserializer();
ArrayList row=(ArrayList)de.deserialize(aoig.getColumnNames(),aoig.getColumnTypes(),garw,s);
assertEquals(1,row.size());
Object theMapObject=row.get(0);
assertTrue(theMapObject instanceof Map);
Map theMap=(Map)theMapObject;
assertEquals(1l,theMap.get("one"));
assertEquals(2l,theMap.get("two"));
assertEquals(3l,theMap.get("three"));
StandardStructObjectInspector oi=(StandardStructObjectInspector)aoig.getObjectInspector();
List z=oi.getStructFieldsDataAsList(row);
assertEquals(1,z.size());
StructField fieldRef=oi.getStructFieldRef("amap");
Map theMap2=(Map)oi.getStructFieldData(row,fieldRef);
assertEquals(1l,theMap2.get("one"));
assertEquals(2l,theMap2.get("two"));
assertEquals(3l,theMap2.get("three"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.avro.TestAvroObjectInspectorGenerator APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier allStructFieldNames=structTypeInfo.getAllStructFieldNames();
ArrayList allStructFieldTypeInfos=structTypeInfo.getAllStructFieldTypeInfos();
assertEquals(allStructFieldNames.size(),3);
String[] names=new String[]{"int1","boolean1","long1"};
String[] typeInfoStrings=new String[]{"int","boolean","bigint"};
for (int i=0; i < allStructFieldNames.size(); i++) {
assertEquals("Fieldname " + allStructFieldNames.get(i) + " doesn't match expected "+ names[i],names[i],allStructFieldNames.get(i));
assertEquals("Typeinfo " + allStructFieldTypeInfos.get(i) + " doesn't match expected "+ typeInfoStrings[i],typeInfoStrings[i],allStructFieldTypeInfos.get(i).getTypeName());
}
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier typeInfos=uti.getAllUnionObjectTypeInfos();
assertEquals(2,typeInfos.size());
assertEquals(INT,typeInfos.get(0));
assertEquals(STRING,typeInfos.get(1));
assertEquals("uniontype",uti.getTypeName());
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.avro.TestAvroSerde EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.avro.TestAvroSerdeUtils APIUtilityVerifier InternalCallVerifier EqualityVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.avro.TestAvroSerializer APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier intList=new ArrayList();
Collections.addAll(intList,1,2,null,3);
GenericRecord r=serializeAndDeserialize(field,"listWithNulls",intList);
Object result=r.get("listWithNulls");
assertNotSame(intList,result);
assertEquals(intList,result);
}
APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier intList=new ArrayList();
Collections.addAll(intList,1,2,3);
String field="{ \"name\":\"nullableList\", \"type\": [\"null\", " + "{\"type\":\"array\", \"items\":\"int\"}] }";
GenericRecord r=serializeAndDeserialize(field,"nullableList",intList);
Object result=r.get("nullableList");
assertNotSame(intList,result);
assertEquals(intList,result);
r=serializeAndDeserialize(field,"nullableList",null);
assertNull(r.get("nullableList"));
}
APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier > intListList=new ArrayList>();
List intList=new ArrayList();
Collections.addAll(intList,1,2,3);
Collections.addAll(intListList,intList,null);
GenericRecord r=serializeAndDeserialize(field,"listOfNullableLists",intListList);
Object result=r.get("listOfNullableLists");
assertNotSame(intListList,result);
assertEquals(intListList,result);
}
APIUtilityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier m=new HashMap();
m.put("yes",true);
m.put("no",false);
m.put("maybe",null);
GenericRecord r=serializeAndDeserialize(field,"mapWithNulls",m);
Object result=r.get("mapWithNulls");
assertEquals(m,result);
}
APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier intList=new ArrayList();
Collections.addAll(intList,1,2,3);
innerRecord.put("nullableList",intList);
GenericRecord r=serializeAndDeserialize(field,"struct1",innerRecord);
Object result=r.get("struct1");
assertNotSame(innerRecord,result);
assertEquals(innerRecord,result);
innerRecord.put("nullableList",null);
r=serializeAndDeserialize(field,"struct1",innerRecord);
result=r.get("struct1");
assertNotSame(innerRecord,result);
assertEquals(innerRecord,result);
}
APIUtilityVerifier EqualityVerifier m=new HashMap();
m.put("yes",true);
m.put("no",false);
String field="{ \"name\":\"map1\", \"type\":{\"type\":\"map\", \"values\":\"boolean\"} }";
GenericRecord r=serializeAndDeserialize(field,"map1",m);
assertEquals(m,r.get("map1"));
}
APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier m=new HashMap();
m.put("yes",true);
m.put("no",false);
GenericRecord r=serializeAndDeserialize(field,"nullableMap",m);
Object result=r.get("nullableMap");
assertNotSame(m,result);
assertEquals(m,result);
r=serializeAndDeserialize(field,"nullableMap",null);
assertNull(r.get("nullableMap"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier columnNames=aoig.getColumnNames();
List columnTypes=aoig.getColumnTypes();
AvroGenericRecordWritable agrw=new AvroGenericRecordWritable(r);
agrw.setFileSchema(r.getSchema());
Object obj=ad.deserialize(columnNames,columnTypes,agrw,s);
Writable result=as.serialize(obj,oi,columnNames,columnTypes,s);
assertTrue(result instanceof AvroGenericRecordWritable);
GenericRecord r2=((AvroGenericRecordWritable)result).getRecord();
assertEquals(s,r2.getSchema());
GenericRecord r3=(GenericRecord)r2.get("struct1");
assertEquals(77,r3.get("sInt"));
assertEquals(false,r3.get("sBoolean"));
assertEquals("tedious",r3.get("sString"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier columnNames=aoig.getColumnNames();
List columnTypes=aoig.getColumnTypes();
AvroGenericRecordWritable agrw=Utils.serializeAndDeserializeRecord(parentRec);
Object obj=ad.deserialize(columnNames,columnTypes,agrw,parentS);
Writable result=as.serialize(obj,oi,columnNames,columnTypes,parentS);
assertTrue(result instanceof AvroGenericRecordWritable);
GenericRecord r2=((AvroGenericRecordWritable)result).getRecord();
assertEquals(parentS,r2.getSchema());
}
APIUtilityVerifier EqualityVerifier m=new HashMap();
m.put("yes",AvroSerdeUtils.getBufferFromDecimal(HiveDecimal.create("3.14"),4));
m.put("no",AvroSerdeUtils.getBufferFromDecimal(HiveDecimal.create("6.2832732"),4));
String field="{ \"name\":\"map1\", \"type\":{\"type\":\"map\"," + " \"values\":{\"type\":\"bytes\", \"logicalType\":\"decimal\", \"precision\":5, \"scale\":4}} }";
GenericRecord r=serializeAndDeserialize(field,"map1",m);
assertEquals(m,r.get("map1"));
}
APIUtilityVerifier EqualityVerifier bbList=new ArrayList();
String[] decs=new String[]{"3.1416","4.7779","0.2312","9.1000","5.5555"};
for (int i=0; i < decs.length; i++) {
bbList.add(AvroSerdeUtils.getBufferFromDecimal(HiveDecimal.create(decs[i]),4));
}
String field="{ \"name\":\"list1\", \"type\":{\"type\":\"array\"," + " \"items\":{\"type\":\"bytes\", \"logicalType\":\"decimal\", \"precision\":5, \"scale\":4}} }";
GenericRecord r=serializeAndDeserialize(field,"list1",bbList);
assertEquals(bbList,r.get("list1"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier IdentityVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier > m=new HashMap>();
List intList=new ArrayList();
Collections.addAll(intList,1,2,3);
m.put("list",intList);
m.put("null",null);
GenericRecord r=serializeAndDeserialize(field,"mapWithNullableLists",m);
Object result=r.get("mapWithNullableLists");
assertNotSame(m,result);
assertEquals(m,result);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier intList=new ArrayList();
Collections.addAll(intList,1,2,3);
String field="{ \"name\":\"list1\", \"type\":{\"type\":\"array\", \"items\":\"int\"} }";
GenericRecord r=serializeAndDeserialize(field,"list1",intList);
final Object list1=r.get("list1");
Assert.assertTrue(list1 instanceof GenericArray);
Assert.assertTrue(list1 instanceof List);
assertEquals(intList,list1);
}
Class: org.apache.hadoop.hive.serde2.avro.TestGenericAvroRecordWritable APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.avro.TestSchemaReEncoder APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.avro.TestThatEvolvedSchemasActAsWeWant APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier gdw=new GenericDatumWriter(schemas[0]);
DataFileWriter dfw=new DataFileWriter(gdw);
ByteArrayOutputStream baos=new ByteArrayOutputStream();
dfw.create(schemas[0],baos);
dfw.append(record);
dfw.close();
ByteArrayInputStream bais=new ByteArrayInputStream(baos.toByteArray());
GenericDatumReader gdr=new GenericDatumReader();
gdr.setExpected(schemas[1]);
DataFileStream dfs=new DataFileStream(bais,gdr);
assertTrue(dfs.hasNext());
GenericRecord next=dfs.next();
assertEquals("v0 value",next.get("v0").toString());
assertEquals("v1_default",next.get("v1").toString());
assertEquals(schemas[1],next.getSchema());
}
Class: org.apache.hadoop.hive.serde2.avro.TestTypeInfoToSchema APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier names=new ArrayList();
names.add("field1");
names.add("field2");
names.add("field3");
names.add("field4");
names.add("field5");
names.add("field6");
names.add("field7");
names.add("field8");
names.add("field9");
names.add("field10");
names.add("field11");
names.add("field12");
names.add("field13");
names.add("field14");
structTypeInfo.setAllStructFieldNames(names);
ArrayList typeInfos=new ArrayList();
typeInfos.add(STRING);
typeInfos.add(CHAR);
typeInfos.add(VARCHAR);
typeInfos.add(BINARY);
typeInfos.add(BYTE);
typeInfos.add(SHORT);
typeInfos.add(INT);
typeInfos.add(LONG);
typeInfos.add(FLOAT);
typeInfos.add(DOUBLE);
typeInfos.add(BOOLEAN);
typeInfos.add(DECIMAL);
typeInfos.add(DATE);
typeInfos.add(VOID);
structTypeInfo.setAllStructFieldTypeInfos(typeInfos);
LOGGER.info("structTypeInfo is " + structTypeInfo);
final String specificSchema=IOUtils.toString(Resources.getResource("avro-struct.avsc").openStream()).replace(lineSeparator,"");
String expectedSchema=genSchema(specificSchema);
Assert.assertEquals("Test for struct's avro schema failed",expectedSchema,getAvroSchemaString(structTypeInfo));
}
APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier names=new ArrayList();
names.add("field1");
names.add("field2");
structTypeInfo.setAllStructFieldNames(names);
ArrayList typeInfos=new ArrayList();
typeInfos.add(STRING);
typeInfos.add(INT);
structTypeInfo.setAllStructFieldTypeInfos(typeInfos);
StructTypeInfo superStructTypeInfo=new StructTypeInfo();
ArrayList superNames=new ArrayList();
superNames.add("superfield1");
superNames.add("superfield2");
superStructTypeInfo.setAllStructFieldNames(superNames);
ArrayList superTypeInfos=new ArrayList();
superTypeInfos.add(STRING);
superTypeInfos.add(structTypeInfo);
superStructTypeInfo.setAllStructFieldTypeInfos(superTypeInfos);
final String specificSchema=IOUtils.toString(Resources.getResource("avro-nested-struct.avsc").openStream()).replace(lineSeparator,"");
String expectedSchema=genSchema(specificSchema);
Assert.assertEquals("Test for nested struct's avro schema failed",expectedSchema,getAvroSchemaString(superStructTypeInfo));
}
APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.columnar.TestBytesRefArrayWritable InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.io.TestDateWritable InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier NullVerifier HybridVerifier future=threadPool.submit(new DateTestCallable());
String result=future.get();
assertNull("Failed at timezone " + timeZone + ", date "+ result,result);
}
finally {
threadPool.shutdown();
TimeZone.setDefault(previousDefault);
}
}
}
Class: org.apache.hadoop.hive.serde2.io.TestHiveCharWritable BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.io.TestHiveDecimalWritable InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.io.TestHiveIntervalDayTimeWritable InternalCallVerifier EqualityVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.io.TestHiveIntervalYearMonthWritable BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.io.TestHiveVarcharWritable InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hadoop.hive.serde2.io.TestTimestampWritable EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.lazy.TestLazySimpleSerDe APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hadoop.hive.serde2.objectinspector.primitive.TestPrimitiveObjectInspectorUtils EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hive.beeline.TestBeeLineHistory APIUtilityVerifier EqualityVerifier Class: org.apache.hive.beeline.TestBeelineArgParsing BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier EqualityVerifier BranchVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.common.util.TestBloomFilter APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.common.util.TestMurmur3 APIUtilityVerifier IterativeVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.hive.common.util.TestShutdownHookManager BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.hcatalog.api.TestHCatClient APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier dbNames=client.listDatabaseNamesByPattern("*");
assertTrue(dbNames.contains("default"));
assertTrue(dbNames.contains(db));
HCatDatabase testDb=client.getDatabase(db);
assertTrue(testDb.getComment() == null);
assertTrue(testDb.getProperties().size() == 0);
String warehouseDir=System.getProperty("test.warehouse.dir","/user/hive/warehouse");
if (useExternalMS) {
assertTrue(testDb.getLocation().matches(".*" + "/" + db + ".db"));
}
else {
String expectedDir=warehouseDir.replaceFirst("pfile:///","pfile:/");
assertEquals(expectedDir + "/" + db+ ".db",testDb.getLocation());
}
ArrayList cols=new ArrayList();
cols.add(new HCatFieldSchema("id",Type.INT,"id comment"));
cols.add(new HCatFieldSchema("value",Type.STRING,"value comment"));
HCatCreateTableDesc tableDesc=HCatCreateTableDesc.create(db,tableOne,cols).fileFormat("rcfile").build();
client.createTable(tableDesc);
HCatTable table1=client.getTable(db,tableOne);
assertTrue(table1.getInputFileFormat().equalsIgnoreCase(RCFileInputFormat.class.getName()));
assertTrue(table1.getOutputFileFormat().equalsIgnoreCase(RCFileOutputFormat.class.getName()));
assertTrue(table1.getSerdeLib().equalsIgnoreCase(LazyBinaryColumnarSerDe.class.getName()));
assertTrue(table1.getCols().equals(cols));
try {
client.createTable(tableDesc);
fail("Expected exception");
}
catch ( HCatException e) {
assertTrue(e.getMessage().contains("AlreadyExistsException while creating table."));
}
client.dropTable(db,tableOne,true);
HCatCreateTableDesc tableDesc2=HCatCreateTableDesc.create(db,tableTwo,cols).fieldsTerminatedBy('\001').escapeChar('\002').linesTerminatedBy('\003').mapKeysTerminatedBy('\004').collectionItemsTerminatedBy('\005').nullDefinedAs('\006').build();
client.createTable(tableDesc2);
HCatTable table2=client.getTable(db,tableTwo);
assertTrue("Expected TextInputFormat, but got: " + table2.getInputFileFormat(),table2.getInputFileFormat().equalsIgnoreCase(TextInputFormat.class.getName()));
assertTrue(table2.getOutputFileFormat().equalsIgnoreCase(HiveIgnoreKeyTextOutputFormat.class.getName()));
assertTrue("SerdeParams not found",table2.getSerdeParams() != null);
assertEquals("checking " + serdeConstants.FIELD_DELIM,Character.toString('\001'),table2.getSerdeParams().get(serdeConstants.FIELD_DELIM));
assertEquals("checking " + serdeConstants.ESCAPE_CHAR,Character.toString('\002'),table2.getSerdeParams().get(serdeConstants.ESCAPE_CHAR));
assertEquals("checking " + serdeConstants.LINE_DELIM,Character.toString('\003'),table2.getSerdeParams().get(serdeConstants.LINE_DELIM));
assertEquals("checking " + serdeConstants.MAPKEY_DELIM,Character.toString('\004'),table2.getSerdeParams().get(serdeConstants.MAPKEY_DELIM));
assertEquals("checking " + serdeConstants.COLLECTION_DELIM,Character.toString('\005'),table2.getSerdeParams().get(serdeConstants.COLLECTION_DELIM));
assertEquals("checking " + serdeConstants.SERIALIZATION_NULL_FORMAT,Character.toString('\006'),table2.getSerdeParams().get(serdeConstants.SERIALIZATION_NULL_FORMAT));
assertTrue(table2.getLocation().toLowerCase().matches(".*" + ("/" + db + ".db/"+ tableTwo).toLowerCase()));
HCatCreateTableDesc tableDesc3=HCatCreateTableDesc.create(db,tableThree,cols).fileFormat("orcfile").build();
client.createTable(tableDesc3);
HCatTable table3=client.getTable(db,tableThree);
assertTrue(table3.getInputFileFormat().equalsIgnoreCase(OrcInputFormat.class.getName()));
assertTrue(table3.getOutputFileFormat().equalsIgnoreCase(OrcOutputFormat.class.getName()));
assertTrue(table3.getSerdeLib().equalsIgnoreCase(OrcSerde.class.getName()));
assertTrue(table1.getCols().equals(cols));
client.close();
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier columnSchema=Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,""));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
client.createTable(HCatCreateTableDesc.create(dbName,tableName,columnSchema).partCols(partitionSchema).build());
HCatTable table=client.getTable(dbName,tableName);
List partitionColumns=table.getPartCols();
assertArrayEquals("Didn't get expected partition-schema back from the HCatTable.",partitionSchema.toArray(),partitionColumns.toArray());
client.dropDatabase(dbName,false,HCatClient.DropDBMode.CASCADE);
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception!",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnSchema=new ArrayList(Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,"")));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
HCatTable sourceTable=new HCatTable(dbName,tableName).cols(columnSchema).partCols(partitionSchema).comment("Source table.");
sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build());
sourceTable=sourceMetaStore.getTable(dbName,tableName);
assertNotNull("Table couldn't be queried for. ",sourceTable);
Map partitionSpec_1=new HashMap();
partitionSpec_1.put("grid","AB");
partitionSpec_1.put("dt","2011_12_31");
HCatPartition sourcePartition_1=new HCatPartition(sourceTable,partitionSpec_1,makePartLocation(sourceTable,partitionSpec_1));
sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build());
assertEquals("Unexpected number of partitions. ",1,sourceMetaStore.getPartitions(dbName,tableName).size());
HCatPartition addedPartition_1=sourceMetaStore.getPartition(dbName,tableName,partitionSpec_1);
assertEquals("Column schema doesn't match.",sourceTable.getCols(),addedPartition_1.getColumns());
assertEquals("InputFormat doesn't match.",sourceTable.getInputFileFormat(),addedPartition_1.getInputFormat());
assertEquals("OutputFormat doesn't match.",sourceTable.getOutputFileFormat(),addedPartition_1.getOutputFormat());
assertEquals("SerDe doesn't match.",sourceTable.getSerdeLib(),addedPartition_1.getSerDe());
assertEquals("SerDe params don't match.",sourceTable.getSerdeParams(),addedPartition_1.getSerdeParams());
HCatClient targetMetaStore=HCatClient.create(new Configuration(replicationTargetHCatConf));
targetMetaStore.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build());
HCatTable targetTable=targetMetaStore.deserializeTable(sourceMetaStore.serializeTable(sourceTable));
targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build());
targetTable=targetMetaStore.getTable(dbName,tableName);
assertEquals("Created table doesn't match the source.",HCatTable.NO_DIFF,targetTable.diff(sourceTable));
List newColumnSchema=new ArrayList(columnSchema);
newColumnSchema.add(new HCatFieldSchema("goo_new",Type.DOUBLE,""));
Map tableParams=new HashMap(1);
tableParams.put("orc.compress","ZLIB");
sourceTable.cols(newColumnSchema).fileFormat("orcfile").tblProps(tableParams).serdeParam(serdeConstants.FIELD_DELIM,Character.toString('\001'));
sourceMetaStore.updateTableSchema(dbName,tableName,sourceTable);
sourceTable=sourceMetaStore.getTable(dbName,tableName);
Map partitionSpec_2=new HashMap();
partitionSpec_2.put("grid","AB");
partitionSpec_2.put("dt","2012_01_01");
HCatPartition sourcePartition_2=new HCatPartition(sourceTable,partitionSpec_2,makePartLocation(sourceTable,partitionSpec_2));
sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build());
HCatPartitionSpec sourcePartitionSpec=sourceMetaStore.getPartitionSpecs(dbName,tableName,-1);
assertEquals("Unexpected number of source partitions.",2,sourcePartitionSpec.size());
List partitionSpecString=sourceMetaStore.serializePartitionSpec(sourcePartitionSpec);
HCatPartitionSpec targetPartitionSpec=targetMetaStore.deserializePartitionSpec(partitionSpecString);
assertEquals("Could not add the expected number of partitions.",sourcePartitionSpec.size(),targetMetaStore.addPartitionSpec(targetPartitionSpec));
targetPartitionSpec=targetMetaStore.getPartitionSpecs(dbName,tableName,-1);
assertEquals("Could not retrieve the expected number of partitions.",sourcePartitionSpec.size(),targetPartitionSpec.size());
HCatPartitionSpec.HCatPartitionIterator sourceIterator=sourcePartitionSpec.getPartitionIterator();
HCatPartitionSpec.HCatPartitionIterator targetIterator=targetPartitionSpec.getPartitionIterator();
while (targetIterator.hasNext()) {
assertTrue("Fewer target partitions than source.",sourceIterator.hasNext());
HCatPartition sourcePartition=sourceIterator.next();
HCatPartition targetPartition=targetIterator.next();
assertEquals("Column schema doesn't match.",sourcePartition.getColumns(),targetPartition.getColumns());
assertEquals("InputFormat doesn't match.",sourcePartition.getInputFormat(),targetPartition.getInputFormat());
assertEquals("OutputFormat doesn't match.",sourcePartition.getOutputFormat(),targetPartition.getOutputFormat());
assertEquals("SerDe doesn't match.",sourcePartition.getSerDe(),targetPartition.getSerDe());
assertEquals("SerDe params don't match.",sourcePartition.getSerdeParams(),targetPartition.getSerdeParams());
}
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception! ",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnSchema=new ArrayList(Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,"")));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
HCatTable sourceTable=new HCatTable(dbName,tableName).cols(columnSchema).partCols(partitionSchema).comment("Source table.");
sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build());
sourceTable=sourceMetaStore.getTable(dbName,tableName);
assertNotNull("Table couldn't be queried for. ",sourceTable);
Map partitionSpec_1=new HashMap();
partitionSpec_1.put("grid","AB");
partitionSpec_1.put("dt","2011_12_31");
HCatPartition sourcePartition_1=new HCatPartition(sourceTable,partitionSpec_1,makePartLocation(sourceTable,partitionSpec_1));
sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build());
assertEquals("Unexpected number of partitions. ",1,sourceMetaStore.getPartitions(dbName,tableName).size());
HCatPartition addedPartition_1=sourceMetaStore.getPartition(dbName,tableName,partitionSpec_1);
assertEquals("Column schema doesn't match.",sourceTable.getCols(),addedPartition_1.getColumns());
assertEquals("InputFormat doesn't match.",sourceTable.getInputFileFormat(),addedPartition_1.getInputFormat());
assertEquals("OutputFormat doesn't match.",sourceTable.getOutputFileFormat(),addedPartition_1.getOutputFormat());
assertEquals("SerDe doesn't match.",sourceTable.getSerdeLib(),addedPartition_1.getSerDe());
assertEquals("SerDe params don't match.",sourceTable.getSerdeParams(),addedPartition_1.getSerdeParams());
HCatClient targetMetaStore=HCatClient.create(new Configuration(replicationTargetHCatConf));
targetMetaStore.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build());
HCatTable targetTable=targetMetaStore.deserializeTable(sourceMetaStore.serializeTable(sourceTable));
targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build());
targetTable=targetMetaStore.getTable(dbName,tableName);
assertEquals("Created table doesn't match the source.",HCatTable.NO_DIFF,targetTable.diff(sourceTable));
List newColumnSchema=new ArrayList(columnSchema);
newColumnSchema.add(new HCatFieldSchema("goo_new",Type.DOUBLE,""));
Map tableParams=new HashMap(1);
tableParams.put("orc.compress","ZLIB");
sourceTable.cols(newColumnSchema).fileFormat("orcfile").tblProps(tableParams).serdeParam(serdeConstants.FIELD_DELIM,Character.toString('\001'));
sourceMetaStore.updateTableSchema(dbName,tableName,sourceTable);
sourceTable=sourceMetaStore.getTable(dbName,tableName);
Map partitionSpec_2=new HashMap();
partitionSpec_2.put("grid","AB");
partitionSpec_2.put("dt","2012_01_01");
HCatPartition sourcePartition_2=new HCatPartition(sourceTable,partitionSpec_2,makePartLocation(sourceTable,partitionSpec_2));
sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build());
List sourcePartitions=sourceMetaStore.getPartitions(dbName,tableName);
assertEquals("Unexpected number of source partitions.",2,sourcePartitions.size());
List addPartitionDescs=new ArrayList(sourcePartitions.size());
for ( HCatPartition partition : sourcePartitions) {
addPartitionDescs.add(HCatAddPartitionDesc.create(partition).build());
}
targetMetaStore.addPartitions(addPartitionDescs);
List targetPartitions=targetMetaStore.getPartitions(dbName,tableName);
assertEquals("Expected the same number of partitions. ",sourcePartitions.size(),targetPartitions.size());
for (int i=0; i < targetPartitions.size(); ++i) {
HCatPartition sourcePartition=sourcePartitions.get(i), targetPartition=targetPartitions.get(i);
assertEquals("Column schema doesn't match.",sourcePartition.getColumns(),targetPartition.getColumns());
assertEquals("InputFormat doesn't match.",sourcePartition.getInputFormat(),targetPartition.getInputFormat());
assertEquals("OutputFormat doesn't match.",sourcePartition.getOutputFormat(),targetPartition.getOutputFormat());
assertEquals("SerDe doesn't match.",sourcePartition.getSerDe(),targetPartition.getSerDe());
assertEquals("SerDe params don't match.",sourcePartition.getSerdeParams(),targetPartition.getSerdeParams());
}
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception! ",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnSchema=Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,""));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
HCatTable table=new HCatTable(dbName,tableName).cols(columnSchema).partCols(partitionSchema);
client.createTable(HCatCreateTableDesc.create(table,false).build());
table=client.getTable(dbName,tableName);
assertNotNull("The created just now can't be null.",table);
Map partitionSpec=new HashMap();
partitionSpec.put("grid","AB");
partitionSpec.put("dt","2011_12_31");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("grid","AB");
partitionSpec.put("dt","2012_01_01");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("dt","2012_01_01");
partitionSpec.put("grid","OB");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("dt","2012_01_01");
partitionSpec.put("grid","XB");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
Map partialPartitionSpec=new HashMap();
partialPartitionSpec.put("dt","2012_01_01");
List partitions=client.getPartitions(dbName,tableName,partialPartitionSpec);
assertEquals("Unexpected number of partitions.",3,partitions.size());
assertArrayEquals("Mismatched partition.",new String[]{"2012_01_01","AB"},partitions.get(0).getValues().toArray());
assertArrayEquals("Mismatched partition.",new String[]{"2012_01_01","OB"},partitions.get(1).getValues().toArray());
assertArrayEquals("Mismatched partition.",new String[]{"2012_01_01","XB"},partitions.get(2).getValues().toArray());
client.dropDatabase(dbName,false,HCatClient.DropDBMode.CASCADE);
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception!",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier columns=Arrays.asList(new HCatFieldSchema("col",Type.STRING,""));
ArrayList partitionColumns=new ArrayList(Arrays.asList(new HCatFieldSchema(partitionColumn,Type.STRING,"")));
HCatTable table=new HCatTable(dbName,tableName).cols(columns).partCols(partitionColumns);
client.createTable(HCatCreateTableDesc.create(table,false).build());
HCatTable createdTable=client.getTable(dbName,tableName);
Map partitionSpec=new HashMap();
partitionSpec.put(partitionColumn,"foobar");
try {
client.getPartition(dbName,tableName,partitionSpec);
assertTrue("Expected ObjectNotFoundException.",false);
}
catch ( Exception exception) {
LOG.info("Got exception: ",exception);
assertTrue("Expected ObjectNotFoundException. Got:" + exception.getClass(),exception instanceof ObjectNotFoundException);
}
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(createdTable,partitionSpec,makePartLocation(createdTable,partitionSpec))).build());
assertEquals("Expected empty set of partitions.",0,client.listPartitionsByFilter(dbName,tableName,partitionColumn + " < 'foobar'").size());
try {
partitionSpec.put("NonExistentKey","foobar");
client.getPartition(dbName,tableName,partitionSpec);
assertTrue("Expected HCatException.",false);
}
catch ( Exception exception) {
LOG.info("Got exception: ",exception);
assertTrue("Expected HCatException. Got:" + exception.getClass(),exception instanceof HCatException);
assertFalse("Did not expect ObjectNotFoundException.",exception instanceof ObjectNotFoundException);
}
}
catch ( Throwable t) {
LOG.error("Unexpected exception!",t);
assertTrue("Unexpected exception! " + t.getMessage(),false);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new HCatFieldSchema("id",Type.INT,"id columns"));
cols.add(new HCatFieldSchema("value",Type.STRING,"id columns"));
try {
HCatCreateTableDesc tableDesc=HCatCreateTableDesc.create(null,tableName,cols).fileFormat("rcfile").build();
client.createTable(tableDesc);
}
catch ( Exception exp) {
isExceptionCaught=true;
assertEquals("Unexpected exception type.",HCatException.class,exp.getClass());
client=HCatClient.create(new Configuration(hcatConf));
String newName="goodTable";
client.dropTable(null,newName,true);
HCatCreateTableDesc tableDesc2=HCatCreateTableDesc.create(null,newName,cols).fileFormat("rcfile").build();
client.createTable(tableDesc2);
HCatTable newTable=client.getTable(null,newName);
assertTrue(newTable != null);
assertTrue(newTable.getTableName().equalsIgnoreCase(newName));
}
finally {
client.close();
assertTrue("The expected exception was never thrown.",isExceptionCaught);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier tableProperties=new HashMap(1);
tableProperties.put(HCatConstants.HCAT_MSGBUS_TOPIC_NAME,messageBusTopicName);
client.createTable(HCatCreateTableDesc.create(dbName,tableName,Arrays.asList(new HCatFieldSchema("foo",Type.STRING,""))).tblProps(tableProperties).build());
assertEquals("MessageBus topic-name doesn't match!",messageBusTopicName,client.getMessageBusTopicName(dbName,tableName));
client.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
client.close();
}
catch ( Exception exception) {
LOG.error("Unexpected exception.",exception);
assertTrue("Unexpected exception:" + exception.getMessage(),false);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier oldSchema=Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,""));
client.createTable(HCatCreateTableDesc.create(dbName,tableName,oldSchema).build());
List newSchema=Arrays.asList(new HCatFieldSchema("completely",Type.DOUBLE,""),new HCatFieldSchema("new",Type.STRING,""),new HCatFieldSchema("fields",Type.STRING,""));
client.updateTableSchema(dbName,tableName,newSchema);
assertArrayEquals(newSchema.toArray(),client.getTable(dbName,tableName).getCols().toArray());
client.dropDatabase(dbName,false,HCatClient.DropDBMode.CASCADE);
}
catch ( Exception exception) {
LOG.error("Unexpected exception.",exception);
assertTrue("Unexpected exception: " + exception.getMessage(),false);
}
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnSchema=Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,""));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
HCatTable sourceTable=new HCatTable(dbName,tableName).cols(columnSchema).partCols(partitionSchema);
sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build());
sourceTable=sourceMetaStore.getTable(dbName,tableName);
assertNotNull("Table couldn't be queried for. ",sourceTable);
String tableStringRep=sourceMetaStore.serializeTable(sourceTable);
HCatClient targetMetaStore=HCatClient.create(new Configuration(replicationTargetHCatConf));
targetMetaStore.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build());
HCatTable targetTable=targetMetaStore.deserializeTable(tableStringRep);
assertEquals("Table after deserialization should have been identical to sourceTable.",HCatTable.NO_DIFF,sourceTable.diff(targetTable));
targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build());
targetTable=targetMetaStore.getTable(dbName,tableName);
assertEquals("Table after deserialization should have been identical to sourceTable.",HCatTable.NO_DIFF,sourceTable.diff(targetTable));
List newColumnSchema=new ArrayList(columnSchema);
newColumnSchema.add(new HCatFieldSchema("goo_new",Type.DOUBLE,""));
Map tableParams=new HashMap(1);
tableParams.put("orc.compress","ZLIB");
sourceTable.cols(newColumnSchema).fileFormat("orcfile").tblProps(tableParams).serdeParam(serdeConstants.FIELD_DELIM,Character.toString('\001'));
sourceMetaStore.updateTableSchema(dbName,tableName,sourceTable);
sourceTable=sourceMetaStore.getTable(dbName,tableName);
EnumSet diff=targetTable.diff(sourceTable);
assertTrue("Couldn't find change in column-schema.",diff.contains(HCatTable.TableAttribute.COLUMNS));
assertTrue("Couldn't find change in InputFormat.",diff.contains(HCatTable.TableAttribute.INPUT_FORMAT));
assertTrue("Couldn't find change in OutputFormat.",diff.contains(HCatTable.TableAttribute.OUTPUT_FORMAT));
assertTrue("Couldn't find change in SerDe.",diff.contains(HCatTable.TableAttribute.SERDE));
assertTrue("Couldn't find change in SerDe parameters.",diff.contains(HCatTable.TableAttribute.SERDE_PROPERTIES));
assertTrue("Couldn't find change in Table parameters.",diff.contains(HCatTable.TableAttribute.TABLE_PROPERTIES));
targetMetaStore.updateTableSchema(dbName,tableName,targetTable.resolve(sourceTable,diff));
targetTable=targetMetaStore.getTable(dbName,tableName);
assertEquals("After propagating schema changes, source and target tables should have been equivalent.",HCatTable.NO_DIFF,targetTable.diff(sourceTable));
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception!",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier columnSchema=Arrays.asList(new HCatFieldSchema("foo",Type.INT,""),new HCatFieldSchema("bar",Type.STRING,""));
List partitionSchema=Arrays.asList(new HCatFieldSchema("dt",Type.STRING,""),new HCatFieldSchema("grid",Type.STRING,""));
HCatTable table=new HCatTable(dbName,tableName).cols(columnSchema).partCols(partitionSchema);
client.createTable(HCatCreateTableDesc.create(table,false).build());
table=client.getTable(dbName,tableName);
assertNotNull("Table couldn't be queried for. ",table);
Map partitionSpec=new HashMap();
partitionSpec.put("grid","AB");
partitionSpec.put("dt","2011_12_31");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("grid","AB");
partitionSpec.put("dt","2012_01_01");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("dt","2012_01_01");
partitionSpec.put("grid","OB");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
partitionSpec.put("dt","2012_01_01");
partitionSpec.put("grid","XB");
client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partitionSpec,makePartLocation(table,partitionSpec))).build());
Map partialPartitionSpec=new HashMap();
partialPartitionSpec.put("dt","2012_01_01");
client.dropPartitions(dbName,tableName,partialPartitionSpec,true);
List partitions=client.getPartitions(dbName,tableName);
assertEquals("Unexpected number of partitions.",1,partitions.size());
assertArrayEquals("Mismatched partition.",new String[]{"2011_12_31","AB"},partitions.get(0).getValues().toArray());
List partColumns=partitions.get(0).getPartColumns();
assertEquals(2,partColumns.size());
assertEquals("dt",partColumns.get(0).getName());
assertEquals("grid",partColumns.get(1).getName());
client.dropDatabase(dbName,false,HCatClient.DropDBMode.CASCADE);
}
catch ( Exception unexpected) {
LOG.error("Unexpected exception!",unexpected);
assertTrue("Unexpected exception! " + unexpected.getMessage(),false);
}
}
Class: org.apache.hive.hcatalog.api.TestHCatClientNotification APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(2,events.size());
HCatNotificationEvent event=events.get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_DROP_TABLE_EVENT,event.getEventType());
assertEquals(dbName,event.getDbName());
assertEquals(tableName,event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_TABLE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"hcatdroptable\",\"timestamp\":[0-9]+}"));
}
InternalCallVerifier EqualityVerifier events=hCatClient.getNextNotification(firstEventId,0,filter);
assertEquals(1,events.size());
assertEquals(firstEventId + 3,events.get(0).getEventId());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(2,events.size());
HCatNotificationEvent event=events.get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_DROP_DATABASE_EVENT,event.getEventType());
assertEquals(dbname,event.getDbName());
assertNull(event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_DATABASE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"hcatdropdb\",\"timestamp\":[0-9]+}"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier partSpec=new HashMap(1);
partSpec.put(partColName,partName);
hCatClient.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partSpec,null)).build());
List events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(2,events.size());
HCatNotificationEvent event=events.get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals(tableName,event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"ADD_PARTITION\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"hcataddparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"pc\":\"testpart\"}]}"));
}
InternalCallVerifier EqualityVerifier events=hCatClient.getNextNotification(firstEventId,2,null);
assertEquals(2,events.size());
assertEquals(firstEventId + 1,events.get(0).getEventId());
assertEquals(firstEventId + 2,events.get(1).getEventId());
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(1,events.size());
HCatNotificationEvent event=events.get(0);
assertEquals(firstEventId + 1,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_CREATE_TABLE_EVENT,event.getEventType());
assertEquals(dbName,event.getDbName());
assertEquals("hcatcreatetable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"CREATE_TABLE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"hcatcreatetable\",\"timestamp\":[0-9]+}"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(1,events.size());
HCatNotificationEvent event=events.get(0);
assertEquals(firstEventId + 1,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_CREATE_DATABASE_EVENT,event.getEventType());
assertEquals("myhcatdb",event.getDbName());
assertNull(event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"CREATE_DATABASE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"myhcatdb\",\"timestamp\":[0-9]+}"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier partSpec=new HashMap(1);
partSpec.put(partColName,partName);
hCatClient.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table,partSpec,null)).build());
hCatClient.dropPartitions(dbName,tableName,partSpec,false);
List events=hCatClient.getNextNotification(firstEventId,0,null);
assertEquals(3,events.size());
HCatNotificationEvent event=events.get(2);
assertEquals(firstEventId + 3,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals(tableName,event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_PARTITION\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"hcatdropparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"pc\":\"testpart\"}]}"));
}
InternalCallVerifier EqualityVerifier events=hCatClient.getNextNotification(firstEventId,1,filter);
assertEquals(1,events.size());
assertEquals(firstEventId + 1,events.get(0).getEventId());
}
Class: org.apache.hive.hcatalog.api.repl.commands.TestCommands APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=HCatSchemaUtils.getHCatSchema("b:string").getFields();
client.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
client.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build());
HCatTable table=(new HCatTable(dbName,tableName)).cols(cols).fileFormat("textfile");
client.createTable(HCatCreateTableDesc.create(table).build());
HCatTable t=client.getTable(dbName,tableName);
assertNotNull(t);
String[] data=new String[]{"eleven","twelve"};
HcatTestUtils.createTestDataFile(tempLocation,data);
CommandProcessorResponse ret=driver.run("LOAD DATA LOCAL INPATH '" + tempLocation + "' OVERWRITE INTO TABLE "+ dbName+ "."+ tableName);
assertEquals(ret.getResponseCode() + ":" + ret.getErrorMessage(),null,ret.getException());
CommandProcessorResponse selectRet=driver.run("SELECT * from " + dbName + "."+ tableName);
assertEquals(selectRet.getResponseCode() + ":" + selectRet.getErrorMessage(),null,selectRet.getException());
List values=new ArrayList();
driver.getResults(values);
assertEquals(2,values.size());
assertEquals(data[0],values.get(0));
assertEquals(data[1],values.get(1));
ExportCommand exportCmd=new ExportCommand(dbName,tableName,null,exportLocation,false,evid);
LOG.info("About to run :" + exportCmd.get().get(0));
CommandProcessorResponse ret2=driver.run(exportCmd.get().get(0));
assertEquals(ret2.getResponseCode() + ":" + ret2.getErrorMessage(),null,ret2.getException());
List exportPaths=exportCmd.cleanupLocationsAfterEvent();
assertEquals(1,exportPaths.size());
String metadata=getMetadataContents(exportPaths.get(0));
LOG.info("Export returned the following _metadata contents:");
LOG.info(metadata);
assertTrue(metadata + "did not match \"repl.scope\"=\"all\"",metadata.matches(".*\"repl.scope\":\"all\".*"));
assertTrue(metadata + "has \"repl.last.id\"",metadata.matches(".*\"repl.last.id\":.*"));
ImportCommand importCmd=new ImportCommand(dbName,importedTableName,null,exportLocation,false,evid);
LOG.info("About to run :" + importCmd.get().get(0));
CommandProcessorResponse ret3=driver.run(importCmd.get().get(0));
assertEquals(ret3.getResponseCode() + ":" + ret3.getErrorMessage(),null,ret3.getException());
CommandProcessorResponse selectRet2=driver.run("SELECT * from " + dbName + "."+ importedTableName);
assertEquals(selectRet2.getResponseCode() + ":" + selectRet2.getErrorMessage(),null,selectRet2.getException());
List values2=new ArrayList();
driver.getResults(values2);
assertEquals(2,values2.size());
assertEquals(data[0],values2.get(0));
assertEquals(data[1],values2.get(1));
HCatTable importedTable=client.getTable(dbName,importedTableName);
assertNotNull(importedTable);
assertTrue(importedTable.getTblProps().containsKey("repl.last.id"));
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier exportPaths=noopExportCmd.cleanupLocationsAfterEvent();
assertEquals(1,exportPaths.size());
String metadata=getMetadataContents(exportPaths.get(0));
LOG.info("Export returned the following _metadata contents:");
LOG.info(metadata);
assertTrue(metadata + "did not match \"repl.noop\"=\"true\"",metadata.matches(".*\"repl.noop\":\"true\".*"));
ImportCommand noopImportCmd=new ImportCommand(dbName,tableName,null,exportLocation,false,evid);
LOG.info("About to run :" + noopImportCmd.get().get(0));
CommandProcessorResponse ret2=driver.run(noopImportCmd.get().get(0));
assertEquals(ret2.getResponseCode() + ":" + ret2.getErrorMessage(),null,ret2.getException());
Exception onfe=null;
try {
HCatDatabase d_doesNotExist=client.getDatabase(dbName);
}
catch ( Exception e) {
onfe=e;
}
assertNotNull(onfe);
assertTrue(onfe instanceof ObjectNotFoundException);
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields();
Command testReplicatedDropCmd=new DropTableCommand(dbName,tableName,true,evid);
assertEquals(evid,testReplicatedDropCmd.getEventId());
assertEquals(1,testReplicatedDropCmd.get().size());
assertEquals(true,testReplicatedDropCmd.isRetriable());
assertEquals(false,testReplicatedDropCmd.isUndoable());
CommandTestUtils.testCommandSerialization(testReplicatedDropCmd);
Command testNormalDropCmd=new DropTableCommand(dbName,tableName,false,evid);
assertEquals(evid,testNormalDropCmd.getEventId());
assertEquals(1,testNormalDropCmd.get().size());
assertEquals(true,testNormalDropCmd.isRetriable());
assertEquals(false,testNormalDropCmd.isUndoable());
CommandTestUtils.testCommandSerialization(testNormalDropCmd);
client.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
client.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build());
Map tprops=new HashMap();
tprops.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid + 5));
HCatTable tableToCreate=(new HCatTable(dbName,tableName)).tblProps(tprops).cols(cols);
client.createTable(HCatCreateTableDesc.create(tableToCreate).build());
HCatTable t1=client.getTable(dbName,tableName);
assertNotNull(t1);
LOG.info("About to run :" + testReplicatedDropCmd.get().get(0));
driver.run(testReplicatedDropCmd.get().get(0));
HCatTable t2=client.getTable(dbName,tableName);
assertNotNull(t2);
LOG.info("About to run :" + testNormalDropCmd.get().get(0));
driver.run(testNormalDropCmd.get().get(0));
Exception onfe=null;
try {
HCatTable t_del=client.getTable(dbName,tableName);
}
catch ( Exception e) {
onfe=e;
}
assertNotNull(onfe);
assertTrue(onfe instanceof ObjectNotFoundException);
Map tprops2=new HashMap();
tprops2.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid - 5));
HCatTable tableToCreate2=(new HCatTable(dbName,tableName)).tblProps(tprops2).cols(cols);
client.createTable(HCatCreateTableDesc.create(tableToCreate2).build());
HCatTable t3=client.getTable(dbName,tableName);
assertNotNull(t3);
LOG.info("About to run :" + testReplicatedDropCmd.get().get(0));
driver.run(testReplicatedDropCmd.get().get(0));
Exception onfe2=null;
try {
HCatTable t_del=client.getTable(dbName,tableName);
}
catch ( Exception e) {
onfe2=e;
}
assertNotNull(onfe2);
assertTrue(onfe2 instanceof ObjectNotFoundException);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=HCatSchemaUtils.getHCatSchema("b:string").getFields();
client.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
client.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build());
HCatTable table=(new HCatTable(dbName,tableName)).cols(cols).fileFormat("textfile");
client.createTable(HCatCreateTableDesc.create(table).build());
HCatTable t=client.getTable(dbName,tableName);
assertNotNull(t);
String[] data=new String[]{"eleven","twelve"};
HcatTestUtils.createTestDataFile(tempLocation,data);
CommandProcessorResponse ret=driver.run("LOAD DATA LOCAL INPATH '" + tempLocation + "' OVERWRITE INTO TABLE "+ dbName+ "."+ tableName);
assertEquals(ret.getResponseCode() + ":" + ret.getErrorMessage(),null,ret.getException());
CommandProcessorResponse selectRet=driver.run("SELECT * from " + dbName + "."+ tableName);
assertEquals(selectRet.getResponseCode() + ":" + selectRet.getErrorMessage(),null,selectRet.getException());
List values=new ArrayList();
driver.getResults(values);
assertEquals(2,values.size());
assertEquals(data[0],values.get(0));
assertEquals(data[1],values.get(1));
ExportCommand exportMdCmd=new ExportCommand(dbName,tableName,null,exportLocation,true,evid);
LOG.info("About to run :" + exportMdCmd.get().get(0));
CommandProcessorResponse ret2=driver.run(exportMdCmd.get().get(0));
assertEquals(ret2.getResponseCode() + ":" + ret2.getErrorMessage(),null,ret2.getException());
List exportPaths=exportMdCmd.cleanupLocationsAfterEvent();
assertEquals(1,exportPaths.size());
String metadata=getMetadataContents(exportPaths.get(0));
LOG.info("Export returned the following _metadata contents:");
LOG.info(metadata);
assertTrue(metadata + "did not match \"repl.scope\"=\"metadata\"",metadata.matches(".*\"repl.scope\":\"metadata\".*"));
assertTrue(metadata + "has \"repl.last.id\"",metadata.matches(".*\"repl.last.id\":.*"));
ImportCommand importMdCmd=new ImportCommand(dbName,importedTableName,null,exportLocation,true,evid);
LOG.info("About to run :" + importMdCmd.get().get(0));
CommandProcessorResponse ret3=driver.run(importMdCmd.get().get(0));
assertEquals(ret3.getResponseCode() + ":" + ret3.getErrorMessage(),null,ret3.getException());
CommandProcessorResponse selectRet2=driver.run("SELECT * from " + dbName + "."+ importedTableName);
assertEquals(selectRet2.getResponseCode() + ":" + selectRet2.getErrorMessage(),null,selectRet2.getException());
List values2=new ArrayList();
driver.getResults(values2);
assertEquals(0,values2.size());
HCatTable importedTable=client.getTable(dbName,importedTableName);
assertNotNull(importedTable);
assertTrue(importedTable.getTblProps().containsKey("repl.last.id"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier pcols=HCatSchemaUtils.getHCatSchema("b:string").getFields();
List cols=HCatSchemaUtils.getHCatSchema("a:int").getFields();
Map ptnDesc=new HashMap();
ptnDesc.put("b","test");
Command testReplicatedDropPtnCmd=new DropPartitionCommand(dbName,tableName,ptnDesc,true,evid);
assertEquals(evid,testReplicatedDropPtnCmd.getEventId());
assertEquals(1,testReplicatedDropPtnCmd.get().size());
assertEquals(true,testReplicatedDropPtnCmd.isRetriable());
assertEquals(false,testReplicatedDropPtnCmd.isUndoable());
CommandTestUtils.testCommandSerialization(testReplicatedDropPtnCmd);
Command testNormalDropPtnCmd=new DropPartitionCommand(dbName,tableName,ptnDesc,false,evid);
assertEquals(evid,testNormalDropPtnCmd.getEventId());
assertEquals(1,testNormalDropPtnCmd.get().size());
assertEquals(true,testNormalDropPtnCmd.isRetriable());
assertEquals(false,testNormalDropPtnCmd.isUndoable());
CommandTestUtils.testCommandSerialization(testNormalDropPtnCmd);
client.dropDatabase(dbName,true,HCatClient.DropDBMode.CASCADE);
client.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build());
Map props=new HashMap();
props.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid + 5));
HCatTable table=(new HCatTable(dbName,tableName)).tblProps(props).cols(cols).partCols(pcols);
client.createTable(HCatCreateTableDesc.create(table).build());
HCatTable tableCreated=client.getTable(dbName,tableName);
assertNotNull(tableCreated);
HCatPartition ptnToAdd=(new HCatPartition(tableCreated,ptnDesc,TestHCatClient.makePartLocation(tableCreated,ptnDesc))).parameters(props);
client.addPartition(HCatAddPartitionDesc.create(ptnToAdd).build());
HCatPartition p1=client.getPartition(dbName,tableName,ptnDesc);
assertNotNull(p1);
LOG.info("About to run :" + testReplicatedDropPtnCmd.get().get(0));
driver.run(testReplicatedDropPtnCmd.get().get(0));
HCatPartition p2=client.getPartition(dbName,tableName,ptnDesc);
assertNotNull(p2);
LOG.info("About to run :" + testNormalDropPtnCmd.get().get(0));
driver.run(testNormalDropPtnCmd.get().get(0));
Exception onfe=null;
try {
HCatPartition p_del=client.getPartition(dbName,tableName,ptnDesc);
}
catch ( Exception e) {
onfe=e;
}
assertNotNull(onfe);
assertTrue(onfe instanceof ObjectNotFoundException);
Map props2=new HashMap();
props2.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid - 5));
HCatPartition ptnToAdd2=(new HCatPartition(tableCreated,ptnDesc,TestHCatClient.makePartLocation(tableCreated,ptnDesc))).parameters(props2);
client.addPartition(HCatAddPartitionDesc.create(ptnToAdd2).build());
HCatPartition p3=client.getPartition(dbName,tableName,ptnDesc);
assertNotNull(p3);
LOG.info("About to run :" + testReplicatedDropPtnCmd.get().get(0));
driver.run(testReplicatedDropPtnCmd.get().get(0));
Exception onfe2=null;
try {
HCatPartition p_del=client.getPartition(dbName,tableName,ptnDesc);
}
catch ( Exception e) {
onfe2=e;
}
assertNotNull(onfe2);
assertTrue(onfe2 instanceof ObjectNotFoundException);
}
Class: org.apache.hive.hcatalog.api.repl.commands.TestNoopCommand InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.api.repl.exim.TestEximReplicationTasks APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier pkeys=HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
t.setPartitionKeys(pkeys);
List addedPtns=new ArrayList();
addedPtns.add(createPtn(t,Arrays.asList("120","abc")));
addedPtns.add(createPtn(t,Arrays.asList("201","xyz")));
NotificationEvent event=new NotificationEvent(getEventId(),getTime(),HCatConstants.HCAT_ADD_PARTITION_EVENT,msgFactory.buildAddPartitionMessage(t,addedPtns.iterator()).toString());
event.setDbName(t.getDbName());
event.setTableName(t.getTableName());
HCatNotificationEvent hev=new HCatNotificationEvent(event);
ReplicationTask rtask=ReplicationTask.create(client,hev);
assertEquals(hev.toString(),rtask.getEvent().toString());
verifyAddPartitionReplicationTask(rtask,t,addedPtns);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier pkeys=HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
t.setPartitionKeys(pkeys);
Partition p=createPtn(t,Arrays.asList("102","lmn"));
NotificationEvent event=new NotificationEvent(getEventId(),getTime(),HCatConstants.HCAT_ALTER_PARTITION_EVENT,msgFactory.buildAlterPartitionMessage(t,p,p).toString());
event.setDbName(t.getDbName());
event.setTableName(t.getTableName());
HCatNotificationEvent hev=new HCatNotificationEvent(event);
ReplicationTask rtask=ReplicationTask.create(client,hev);
assertEquals(hev.toString(),rtask.getEvent().toString());
verifyAlterPartitionReplicationTask(rtask,t,p);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier pkeys=HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
t.setPartitionKeys(pkeys);
Partition p=createPtn(t,Arrays.asList("102","lmn"));
NotificationEvent event=new NotificationEvent(getEventId(),getTime(),HCatConstants.HCAT_DROP_PARTITION_EVENT,msgFactory.buildDropPartitionMessage(t,Collections.singletonList(p).iterator()).toString());
event.setDbName(t.getDbName());
event.setTableName(t.getTableName());
HCatNotificationEvent hev=new HCatNotificationEvent(event);
ReplicationTask rtask=ReplicationTask.create(client,hev);
assertEquals(hev.toString(),rtask.getEvent().toString());
verifyDropPartitionReplicationTask(rtask,t,p);
}
InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier pkeys=HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
t.setPartitionKeys(pkeys);
Partition p=createPtn(t,Arrays.asList("102","lmn"));
List files=Arrays.asList("/tmp/test123");
NotificationEvent event=new NotificationEvent(getEventId(),getTime(),HCatConstants.HCAT_INSERT_EVENT,msgFactory.buildInsertMessage(t.getDbName(),t.getTableName(),getPtnDesc(t,p),files).toString());
event.setDbName(t.getDbName());
event.setTableName(t.getTableName());
HCatNotificationEvent hev=new HCatNotificationEvent(event);
ReplicationTask rtask=ReplicationTask.create(client,hev);
assertEquals(hev.toString(),rtask.getEvent().toString());
verifyInsertReplicationTask(rtask,t,p);
}
Class: org.apache.hive.hcatalog.cli.TestSemanticAnalysis InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=tbl.getSd().getCols();
assertEquals(1,cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a","int",null)));
assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());
CommandProcessorResponse resp=hcatDriver.run("create table if not exists junit_sem_analysis (a int) stored as RCFILE");
assertEquals(0,resp.getResponseCode());
assertNull(resp.getErrorMessage());
tbl=client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,TBL_NAME);
cols=tbl.getSd().getCols();
assertEquals(1,cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a","int",null)));
assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier partVals=new ArrayList(1);
partVals.add("2010-10-10");
Partition part=client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME,TBL_NAME,partVals);
assertEquals(RCFileInputFormat.class.getName(),part.getSd().getInputFormat());
assertEquals(RCFileOutputFormat.class.getName(),part.getSd().getOutputFormat());
hcatDriver.run("drop table junit_sem_analysis");
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier dbs=client.getAllDatabases();
String testDb1="testdatabaseoperatons1";
String testDb2="testdatabaseoperatons2";
if (dbs.contains(testDb1.toLowerCase())) {
assertEquals(0,hcatDriver.run("drop database " + testDb1).getResponseCode());
}
if (dbs.contains(testDb2.toLowerCase())) {
assertEquals(0,hcatDriver.run("drop database " + testDb2).getResponseCode());
}
assertEquals(0,hcatDriver.run("create database " + testDb1).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb1));
assertEquals(0,hcatDriver.run("create database if not exists " + testDb1).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb1));
assertEquals(0,hcatDriver.run("create database if not exists " + testDb2).getResponseCode());
assertTrue(client.getAllDatabases().contains(testDb2));
assertEquals(0,hcatDriver.run("drop database " + testDb1).getResponseCode());
assertEquals(0,hcatDriver.run("drop database " + testDb2).getResponseCode());
assertFalse(client.getAllDatabases().contains(testDb1));
assertFalse(client.getAllDatabases().contains(testDb2));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier result=new ArrayList();
hcatDriver.getResults(result);
assertTrue(result.get(0).contains("mydb"));
hcatDriver.run("drop database mydb cascade");
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier cols=tbl.getSd().getCols();
assertEquals(2,cols.size());
assertTrue(cols.get(0).equals(new FieldSchema("a1","tinyint",null)));
assertTrue(cols.get(1).equals(new FieldSchema("d","tinyint",null)));
hcatDriver.run("drop table junit_sem_analysis");
}
InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.common.TestHCatUtil APIUtilityVerifier IterativeVerifier BranchVerifier EqualityVerifier permsCode=new HashMap();
for (int i=0; i < 8; i++) {
for (int j=0; j < 8; j++) {
for (int k=0; k < 8; k++) {
StringBuilder sb=new StringBuilder();
sb.append("0");
sb.append(i);
sb.append(j);
sb.append(k);
Integer code=(((i * 8) + j) * 8) + k;
String perms=(new FsPermission(Short.decode(sb.toString()))).toString();
if (permsCode.containsKey(perms)) {
Assert.assertEquals("permissions(" + perms + ") mapped to multiple codes",code,permsCode.get(perms));
}
permsCode.put(perms,code);
assertFsPermissionTransformationIsGood(perms);
}
}
}
}
EqualityVerifier
* org.apache.hadoop.hive.metastore.api.Table - exactly whats stored in the metastore
* org.apache.hadoop.hive.ql.metadata.Table - adds business logic over api.Table
*
* Here we check SerDe-reported fields are included in the table schema.
*/
@Test public void testGetTableSchemaWithPtnColsSerDeReportedFields() throws IOException {
Map parameters=Maps.newHashMap();
parameters.put(serdeConstants.SERIALIZATION_CLASS,"org.apache.hadoop.hive.serde2.thrift.test.IntString");
parameters.put(serdeConstants.SERIALIZATION_FORMAT,"org.apache.thrift.protocol.TBinaryProtocol");
SerDeInfo serDeInfo=new SerDeInfo(null,"org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer",parameters);
StorageDescriptor sd=new StorageDescriptor(new ArrayList(),"location","org.apache.hadoop.mapred.TextInputFormat","org.apache.hadoop.mapred.TextOutputFormat",false,-1,serDeInfo,new ArrayList(),new ArrayList(),new HashMap());
org.apache.hadoop.hive.metastore.api.Table apiTable=new org.apache.hadoop.hive.metastore.api.Table("test_tblname","test_dbname","test_owner",0,0,0,sd,new ArrayList(),new HashMap(),"viewOriginalText","viewExpandedText",TableType.EXTERNAL_TABLE.name());
Table table=new Table(apiTable);
List expectedHCatSchema=Lists.newArrayList(new HCatFieldSchema("myint",HCatFieldSchema.Type.INT,null),new HCatFieldSchema("mystring",HCatFieldSchema.Type.STRING,null),new HCatFieldSchema("underscore_int",HCatFieldSchema.Type.INT,null));
Assert.assertEquals(new HCatSchema(expectedHCatSchema),HCatUtil.getTableSchemaWithPtnCols(table));
}
InternalCallVerifier EqualityVerifier (),new ArrayList(),new HashMap());
org.apache.hadoop.hive.metastore.api.Table apiTable=new org.apache.hadoop.hive.metastore.api.Table("test_tblname","test_dbname","test_owner",0,0,0,sd,new ArrayList(),new HashMap(),"viewOriginalText","viewExpandedText",TableType.EXTERNAL_TABLE.name());
Table table=new Table(apiTable);
List expectedHCatSchema=Lists.newArrayList(new HCatFieldSchema("username",HCatFieldSchema.Type.STRING,null));
Assert.assertEquals(new HCatSchema(expectedHCatSchema),HCatUtil.getTableSchemaWithPtnCols(table));
List partitionKeys=Lists.newArrayList(new FieldSchema("dt",serdeConstants.STRING_TYPE_NAME,null));
table.getTTable().setPartitionKeys(partitionKeys);
expectedHCatSchema.add(new HCatFieldSchema("dt",HCatFieldSchema.Type.STRING,null));
Assert.assertEquals(new HCatSchema(expectedHCatSchema),HCatUtil.getTableSchemaWithPtnCols(table));
}
Class: org.apache.hive.hcatalog.common.TestHiveClientCache InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.hcatalog.data.TestLazyHCatRecord InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.hbase.TestPigHBaseStorageHandler APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier itr=server.openIterator("C");
while (itr.hasNext()) {
Tuple t=itr.next();
assertTrue(t.size() == 2);
assertTrue(t.get(0).getClass() == Integer.class);
assertEquals(index,t.get(0));
assertTrue(t.get(1).getClass() == String.class);
assertEquals("textB-" + index,t.get(1));
index++;
}
assertEquals(index - 1,4);
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier fields=dumpedBSchema.getFields();
assertEquals(3,fields.size());
assertEquals(DataType.INTEGER,fields.get(0).type);
assertEquals("key",fields.get(0).alias.toLowerCase());
assertEquals(DataType.FLOAT,fields.get(1).type);
assertEquals("testQualifier1".toLowerCase(),fields.get(1).alias.toLowerCase());
assertEquals(DataType.CHARARRAY,fields.get(2).type);
assertEquals("testQualifier2".toLowerCase(),fields.get(2).alias.toLowerCase());
Configuration conf=new Configuration(getHbaseConf());
HTable table=new HTable(conf,hbaseTableName);
Scan scan=new Scan();
scan.addFamily(Bytes.toBytes("testFamily"));
byte[] familyNameBytes=Bytes.toBytes("testFamily");
ResultScanner scanner=table.getScanner(scan);
int index=3;
int count=0;
for ( Result result : scanner) {
assertEquals(index,Bytes.toInt(result.getRow()));
assertTrue(result.containsColumn(familyNameBytes,Bytes.toBytes("testQualifier1")));
assertEquals((index + f),Bytes.toFloat(result.getValue(familyNameBytes,Bytes.toBytes("testQualifier1"))),0);
assertTrue(result.containsColumn(familyNameBytes,Bytes.toBytes("testQualifier2")));
assertEquals(("textB-" + index).toString(),Bytes.toString(result.getValue(familyNameBytes,Bytes.toBytes("testQualifier2"))));
index++;
count++;
}
assertEquals(count,5);
driver.run(selectQuery);
ArrayList result=new ArrayList();
driver.getResults(result);
assertEquals(5,result.size());
Iterator itr=result.iterator();
for (int i=3; i <= 7; i++) {
String tokens[]=itr.next().split("\\s+");
assertEquals(i,Integer.parseInt(tokens[0]));
assertEquals(i + f,Float.parseFloat(tokens[1]),0);
assertEquals(("textB-" + i).toString(),tokens[2]);
}
CommandProcessorResponse responseFour=driver.run(deleteQuery);
assertEquals(0,responseFour.getResponseCode());
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier fields=dumpedASchema.getFields();
assertEquals(3,fields.size());
assertEquals(DataType.FLOAT,fields.get(0).type);
assertEquals("key",fields.get(0).alias.toLowerCase());
assertEquals(DataType.CHARARRAY,fields.get(1).type);
assertEquals("testQualifier1".toLowerCase(),fields.get(1).alias.toLowerCase());
assertEquals(DataType.INTEGER,fields.get(2).type);
assertEquals("testQualifier2".toLowerCase(),fields.get(2).alias.toLowerCase());
}
Class: org.apache.hive.hcatalog.listener.TestDbNotificationListener InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("dropPartTable","default","me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
msClient.createTable(table);
Partition partition=new Partition(Arrays.asList("today"),"default","dropPartTable",startTime,startTime,sd,emptyParameters);
msClient.add_partition(partition);
msClient.dropPartition("default","dropparttable",Arrays.asList("today"),false);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(3,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(2);
assertEquals(firstEventId + 3,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("dropparttable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_PARTITION\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"dropparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"ds\":\"today\"}]}"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("addPartTable","default","me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
msClient.createTable(table);
Partition partition=new Partition(Arrays.asList("today"),"default","addPartTable",startTime,startTime,sd,emptyParameters);
msClient.add_partition(partition);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(2,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("addparttable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"ADD_PARTITION\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"addparttable\",\"timestamp\":[0-9]+,\"partitions\":\\[\\{\"ds\":\"today\"}]}"));
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("alterparttable","default","me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
msClient.createTable(table);
Partition partition=new Partition(Arrays.asList("today"),"default","alterparttable",startTime,startTime,sd,emptyParameters);
msClient.add_partition(partition);
Partition newPart=new Partition(Arrays.asList("today"),"default","alterparttable",startTime,startTime + 1,sd,emptyParameters);
msClient.alter_partition("default","alterparttable",newPart);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(3,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(2);
assertEquals(firstEventId + 3,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("alterparttable",event.getTableName());
assertTrue(event.getMessage(),event.getMessage().matches("\\{\"eventType\":\"ALTER_PARTITION\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"alterparttable\"," + "\"timestamp\":[0-9]+,\"keyValues\":\\{\"ds\":\"today\"}}"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
List partCols=new ArrayList();
partCols.add(new FieldSchema("ds","string",""));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("insertPartition","default","me",startTime,startTime,0,sd,partCols,emptyParameters,null,null,null);
msClient.createTable(table);
Partition partition=new Partition(Arrays.asList("today"),"default","insertPartition",startTime,startTime,sd,emptyParameters);
msClient.add_partition(partition);
FireEventRequestData data=new FireEventRequestData();
InsertEventRequestData insertData=new InsertEventRequestData();
data.setInsertData(insertData);
insertData.addToFilesAdded("/warehouse/mytable/today/b1");
FireEventRequest rqst=new FireEventRequest(true,data);
rqst.setDbName("default");
rqst.setTableName("insertPartition");
rqst.setPartitionVals(Arrays.asList("today"));
msClient.fireListenerEvent(rqst);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(3,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(2);
assertEquals(firstEventId + 3,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_INSERT_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("insertPartition",event.getTableName());
assertTrue(event.getMessage(),event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"insertPartition\",\"timestamp\":[0-9]+,"+ "\"files\":\\[\"/warehouse/mytable/today/b1\"],\"partKeyVals\":\\{\"ds\":\"today\"},"+ "\"partitionKeyValues\":\\{\"ds\":\"today\"}}"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("mytable","default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
msClient.createTable(table);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(1,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(0);
assertEquals(firstEventId + 1,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_CREATE_TABLE_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("mytable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"CREATE_TABLE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"mytable\",\"timestamp\":[0-9]+}"));
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("droptable","default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
msClient.createTable(table);
msClient.dropTable("default","droptable");
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(2,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_DROP_TABLE_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("droptable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"DROP_TABLE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"droptable\",\"timestamp\":[0-9]+}"));
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("insertTable","default","me",startTime,startTime,0,sd,null,emptyParameters,null,null,null);
msClient.createTable(table);
FireEventRequestData data=new FireEventRequestData();
InsertEventRequestData insertData=new InsertEventRequestData();
data.setInsertData(insertData);
insertData.addToFilesAdded("/warehouse/mytable/b1");
FireEventRequest rqst=new FireEventRequest(true,data);
rqst.setDbName("default");
rqst.setTableName("insertTable");
msClient.fireListenerEvent(rqst);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(2,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_INSERT_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("insertTable",event.getTableName());
assertTrue(event.getMessage(),event.getMessage().matches("\\{\"eventType\":\"INSERT\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":" + "\"insertTable\",\"timestamp\":[0-9]+,\"files\":\\[\"/warehouse/mytable/b1\"],"+ "\"partKeyVals\":\\{},\"partitionKeyValues\":\\{}}"));
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier cols=new ArrayList();
cols.add(new FieldSchema("col1","int","nocomment"));
SerDeInfo serde=new SerDeInfo("serde","seriallib",null);
StorageDescriptor sd=new StorageDescriptor(cols,"file:/tmp","input","output",false,0,serde,null,null,emptyParameters);
Table table=new Table("alttable","default","me",startTime,startTime,0,sd,new ArrayList(),emptyParameters,null,null,null);
msClient.createTable(table);
cols.add(new FieldSchema("col2","int",""));
table=new Table("alttable","default","me",startTime,startTime,0,sd,new ArrayList(),emptyParameters,null,null,null);
msClient.alter_table("default","alttable",table);
NotificationEventResponse rsp=msClient.getNextNotification(firstEventId,0,null);
assertEquals(2,rsp.getEventsSize());
NotificationEvent event=rsp.getEvents().get(1);
assertEquals(firstEventId + 2,event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(HCatConstants.HCAT_ALTER_TABLE_EVENT,event.getEventType());
assertEquals("default",event.getDbName());
assertEquals("alttable",event.getTableName());
assertTrue(event.getMessage().matches("\\{\"eventType\":\"ALTER_TABLE\",\"server\":\"\"," + "\"servicePrincipal\":\"\",\"db\":\"default\",\"table\":\"alttable\"," + "\"timestamp\":[0-9]+}"));
}
Class: org.apache.hive.hcatalog.listener.TestNotificationListener TestCleaner EqualityVerifier HybridVerifier expectedMessages=Arrays.asList(HCatConstants.HCAT_CREATE_DATABASE_EVENT,HCatConstants.HCAT_CREATE_TABLE_EVENT,HCatConstants.HCAT_ADD_PARTITION_EVENT,HCatConstants.HCAT_ALTER_PARTITION_EVENT,HCatConstants.HCAT_DROP_PARTITION_EVENT,HCatConstants.HCAT_ALTER_TABLE_EVENT,HCatConstants.HCAT_DROP_TABLE_EVENT,HCatConstants.HCAT_DROP_DATABASE_EVENT);
Assert.assertEquals(expectedMessages,actualMessages);
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatHiveCompatibility APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier itr=server.openIterator("B");
int i=0;
while (itr.hasNext()) {
Tuple t=itr.next();
Assert.assertEquals(1,t.size());
Assert.assertEquals(t.get(0),i);
i++;
}
Assert.assertFalse(itr.hasNext());
Assert.assertEquals(11,i);
Table table2=client.getTable("default","junit_unparted_noisd");
Assert.assertTrue(table2.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS));
driver.run("drop table junit_unparted_noisd");
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier itr=server.openIterator("B");
int i=0;
while (itr.hasNext()) {
Tuple t=itr.next();
Assert.assertEquals(2,t.size());
Assert.assertEquals(t.get(0),i);
Assert.assertEquals(t.get(1),"42");
i++;
}
Assert.assertFalse(itr.hasNext());
Assert.assertEquals(11,i);
Table table2=client.getTable("default","junit_parted_noisd");
Assert.assertTrue(table2.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS));
Partition ptn=client.getPartition("default","junit_parted_noisd",Arrays.asList("42"));
Assert.assertNotNull(ptn);
Assert.assertTrue(ptn.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS));
driver.run("drop table junit_unparted_noisd");
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatHiveThriftCompatibility APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier iterator=pigServer.openIterator("A");
Tuple t=iterator.next();
Assert.assertEquals(1,t.get(0));
Assert.assertEquals("one",t.get(1));
Assert.assertEquals(1,t.get(2));
Assert.assertEquals("2012",t.get(3));
Assert.assertFalse(iterator.hasNext());
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatInputFormat TestInitializer InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.hcatalog.mapreduce.TestHCatInputFormatMethods TestInitializer EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.hcatalog.mapreduce.TestHCatMultiOutputFormat IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier
* Submits a mapred job which writes out one fixed line to each of the tables
* uses hive fetch task to read the data and see if it matches what was written
*
* @throws Exception if any error occurs
*/
@Test public void testOutputFormat() throws Throwable {
HashMap partitionValues=new HashMap();
partitionValues.put("ds","1");
partitionValues.put("cluster","ag");
ArrayList infoList=new ArrayList();
infoList.add(OutputJobInfo.create("default",tableNames[0],partitionValues));
infoList.add(OutputJobInfo.create("default",tableNames[1],partitionValues));
infoList.add(OutputJobInfo.create("default",tableNames[2],partitionValues));
Job job=new Job(hiveConf,"SampleJob");
job.setMapperClass(MyMapper.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(MultiOutputFormat.class);
job.setNumReduceTasks(0);
JobConfigurer configurer=MultiOutputFormat.createConfigurer(job);
for (int i=0; i < tableNames.length; i++) {
configurer.addOutputFormat(tableNames[i],HCatOutputFormat.class,BytesWritable.class,HCatRecord.class);
HCatOutputFormat.setOutput(configurer.getJob(tableNames[i]),infoList.get(i));
HCatOutputFormat.setSchema(configurer.getJob(tableNames[i]),schemaMap.get(tableNames[i]));
}
configurer.configure();
Path filePath=createInputFile();
FileInputFormat.addInputPath(job,filePath);
Assert.assertTrue(job.waitForCompletion(true));
ArrayList outputs=new ArrayList();
for ( String tbl : tableNames) {
outputs.add(getTableData(tbl,"default").get(0));
}
Assert.assertEquals("Comparing output of table " + tableNames[0] + " is not correct",outputs.get(0),"a,a,1,ag");
Assert.assertEquals("Comparing output of table " + tableNames[1] + " is not correct",outputs.get(1),"a,1,ag");
Assert.assertEquals("Comparing output of table " + tableNames[2] + " is not correct",outputs.get(2),"a,a,extra,1,ag");
for (int i=0; i < tableNames.length; i++) {
Path partitionFile=new Path(warehousedir + "/" + tableNames[i]+ "/ds=1/cluster=ag/part-m-00000");
FileSystem fs=partitionFile.getFileSystem(mrConf);
Assert.assertEquals("File permissions of table " + tableNames[i] + " is not correct",fs.getFileStatus(partitionFile).getPermission(),new FsPermission(tablePerms[i]));
Assert.assertEquals("File permissions of table " + tableNames[i] + " is not correct",fs.getFileStatus(partitionFile.getParent()).getPermission(),new FsPermission(tablePerms[i]));
Assert.assertEquals("File permissions of table " + tableNames[i] + " is not correct",fs.getFileStatus(partitionFile.getParent().getParent()).getPermission(),new FsPermission(tablePerms[i]));
}
LOG.info("File permissions verified");
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatNonPartitioned APIUtilityVerifier BranchVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier partitionMap=new HashMap();
runMRCreate(null,partitionColumns,writeRecords,10,true);
IOException exc=null;
try {
Job j=runMRCreate(null,partitionColumns,writeRecords,20,true);
assertEquals(!isTableImmutable(),j.isSuccessful());
}
catch ( IOException e) {
exc=e;
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_NON_EMPTY_TABLE,((HCatException)exc).getErrorType());
}
if (!isTableImmutable()) {
assertNull(exc);
}
exc=null;
partitionMap.clear();
partitionMap.put("px","p1value2");
try {
Job j=runMRCreate(partitionMap,partitionColumns,writeRecords,20,true);
assertFalse(j.isSuccessful());
}
catch ( IOException e) {
exc=e;
assertTrue(exc != null);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES,((HCatException)exc).getErrorType());
}
if (isTableImmutable()) {
runMRRead(10);
}
else {
runMRRead(30);
}
hiveReadTest();
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish BranchVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier partitionMap=new HashMap();
partitionMap.put("part1","p1value1");
partitionMap.put("part0","p0value1");
ArrayList hcatTableColumns=new ArrayList();
for ( FieldSchema fs : getTableColumns()) {
hcatTableColumns.add(HCatSchemaUtils.getHCatFieldSchema(fs));
}
runMRCreateFail(dbName,tableName,partitionMap,hcatTableColumns);
List ptns=msc.listPartitionNames(dbName,tableName,(short)10);
Assert.assertEquals(0,ptns.size());
Table table=msc.getTable(dbName,tableName);
Assert.assertTrue(table != null);
if (!Shell.WINDOWS) {
Path path=new Path(table.getSd().getLocation() + "/part1=p1value1/part0=p0value1");
Assert.assertFalse(path.getFileSystem(conf).exists(path));
}
}
Class: org.apache.hive.hcatalog.mapreduce.TestHCatPartitioned APIUtilityVerifier BranchVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier partitionMap=new HashMap();
partitionMap.put("part1","p1value1");
partitionMap.put("part0","501");
runMRCreate(partitionMap,partitionColumns,writeRecords,10,true);
partitionMap.clear();
partitionMap.put("PART1","p1value2");
partitionMap.put("PART0","502");
runMRCreate(partitionMap,partitionColumns,writeRecords,20,true);
IOException exc=null;
try {
Job j=runMRCreate(partitionMap,partitionColumns,writeRecords,20,true);
assertEquals(!isTableImmutable(),j.isSuccessful());
}
catch ( IOException e) {
exc=e;
assertTrue(exc instanceof HCatException);
assertTrue(ErrorType.ERROR_DUPLICATE_PARTITION.equals(((HCatException)exc).getErrorType()));
}
if (!isTableImmutable()) {
assertNull(exc);
}
exc=null;
partitionMap.clear();
partitionMap.put("px1","p1value2");
partitionMap.put("px0","502");
try {
Job j=runMRCreate(partitionMap,partitionColumns,writeRecords,20,true);
assertFalse(j.isSuccessful());
}
catch ( IOException e) {
exc=e;
assertNotNull(exc);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_MISSING_PARTITION_KEY,((HCatException)exc).getErrorType());
}
exc=null;
partitionMap.clear();
partitionMap.put("px","512");
try {
runMRCreate(partitionMap,partitionColumns,writeRecords,20,true);
}
catch ( IOException e) {
exc=e;
}
assertNotNull(exc);
assertTrue(exc instanceof HCatException);
assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES,((HCatException)exc).getErrorType());
exc=null;
try {
runMRCreate(null,partitionColumns,writeRecords,20,false);
}
catch ( IOException e) {
exc=e;
}
assertTrue(exc == null);
if (isTableImmutable()) {
runMRRead(30);
}
else {
runMRRead(50);
}
runMRRead(10,"part1 = \"p1value1\"");
runMRRead(10,"part0 = \"501\"");
if (isTableImmutable()) {
runMRRead(20,"part1 = \"p1value2\"");
runMRRead(30,"part1 = \"p1value1\" or part1 = \"p1value2\"");
runMRRead(20,"part0 = \"502\"");
runMRRead(30,"part0 = \"501\" or part0 = \"502\"");
}
else {
runMRRead(40,"part1 = \"p1value2\"");
runMRRead(50,"part1 = \"p1value1\" or part1 = \"p1value2\"");
runMRRead(40,"part0 = \"502\"");
runMRRead(50,"part0 = \"501\" or part0 = \"502\"");
}
tableSchemaTest();
columnOrderChangeTest();
hiveReadTest();
}
Class: org.apache.hive.hcatalog.mapreduce.TestInputJobInfo APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.mapreduce.TestMultiOutputFormat APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier fileClassPathsList=Arrays.asList(fileClassPaths);
Assert.assertTrue("Cannot find " + (new Path(inputFile)) + " in "+ fileClassPathsList,fileClassPathsList.contains(new Path(inputFile)));
Assert.assertTrue("Cannot find " + (new Path(dummyFile)) + " in "+ fileClassPathsList,fileClassPathsList.contains(new Path(dummyFile)));
URI[] cacheFiles=DistributedCache.getCacheFiles(job.getConfiguration());
List cacheFilesList=Arrays.asList(cacheFiles);
URI inputFileURI=new Path(inputFile).makeQualified(fs).toUri();
Assert.assertTrue("Cannot find " + inputFileURI + " in "+ cacheFilesList,cacheFilesList.contains(inputFileURI));
URI dummyFileURI=new Path(dummyFile).makeQualified(fs).toUri();
Assert.assertTrue("Cannot find " + dummyFileURI + " in "+ cacheFilesList,cacheFilesList.contains(dummyFileURI));
Assert.assertTrue(job.waitForCompletion(true));
Path textOutPath=new Path(outDir,"out1/part-m-00000");
String[] textOutput=readFully(textOutPath).split("\n");
Path seqOutPath=new Path(outDir,"out2/part-m-00000");
SequenceFile.Reader reader=new SequenceFile.Reader(fs,seqOutPath,mrConf);
Text key=new Text();
IntWritable value=new IntWritable();
String[] words=fileContent.split(" ");
Assert.assertEquals(words.length,textOutput.length);
LOG.info("Verifying file contents");
for (int i=0; i < words.length; i++) {
Assert.assertEquals((i + 1) + "\t" + words[i],textOutput[i]);
reader.next(key,value);
Assert.assertEquals(words[i],key.toString());
Assert.assertEquals((i + 1),value.get());
}
Assert.assertFalse(reader.next(key,value));
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.hcatalog.mapreduce.TestSequenceFileReadWrite APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier XIter=server.openIterator("B");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(3,t.size());
assertEquals(t.get(0).toString(),"" + numTuplesRead);
assertEquals(t.get(1).toString(),"a" + numTuplesRead);
assertEquals(t.get(2).toString(),"b" + numTuplesRead);
numTuplesRead++;
}
assertEquals(input.length,numTuplesRead);
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier XIter=server.openIterator("C");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(3,t.size());
assertEquals(t.get(0).toString(),"" + numTuplesRead);
assertEquals(t.get(1).toString(),"a" + numTuplesRead);
assertEquals(t.get(2).toString(),"b" + numTuplesRead);
numTuplesRead++;
}
assertEquals(input.length,numTuplesRead);
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier XIter=server.openIterator("D");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(3,t.size());
assertEquals(t.get(0).toString(),"" + numTuplesRead);
assertEquals(t.get(1).toString(),"a" + numTuplesRead);
assertEquals(t.get(2).toString(),"b" + numTuplesRead);
numTuplesRead++;
}
assertEquals(input.length,numTuplesRead);
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier XIter=server.openIterator("B");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(3,t.size());
assertEquals(t.get(0).toString(),"" + numTuplesRead);
assertEquals(t.get(1).toString(),"a" + numTuplesRead);
assertEquals(t.get(2).toString(),"b" + numTuplesRead);
numTuplesRead++;
}
assertEquals(input.length,numTuplesRead);
}
Class: org.apache.hive.hcatalog.pig.TestHCatLoader APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier Y2fields=dumpedY2Schema.getFields();
List Y3fields=dumpedY3Schema.getFields();
assertEquals(1,Y2fields.size());
assertEquals("a",Y2fields.get(0).alias.toLowerCase());
assertEquals(DataType.INTEGER,Y2fields.get(0).type);
assertEquals(2,Y3fields.size());
assertEquals("b",Y3fields.get(0).alias.toLowerCase());
assertEquals(DataType.CHARARRAY,Y3fields.get(0).type);
assertEquals("a",Y3fields.get(1).alias.toLowerCase());
assertEquals(DataType.INTEGER,Y3fields.get(1).type);
int numTuplesRead=0;
Iterator Y2Iter=server.openIterator("Y2");
while (Y2Iter.hasNext()) {
Tuple t=Y2Iter.next();
assertEquals(t.size(),1);
assertNotNull(t.get(0));
assertTrue(t.get(0).getClass() == Integer.class);
assertEquals(t.get(0),basicInputData.get(numTuplesRead).first);
numTuplesRead++;
}
numTuplesRead=0;
Iterator Y3Iter=server.openIterator("Y3");
while (Y3Iter.hasNext()) {
Tuple t=Y3Iter.next();
assertEquals(t.size(),2);
assertNotNull(t.get(0));
assertTrue(t.get(0).getClass() == String.class);
assertEquals(t.get(0),basicInputData.get(numTuplesRead).second);
assertNotNull(t.get(1));
assertTrue(t.get(1).getClass() == Integer.class);
assertEquals(t.get(1),basicInputData.get(numTuplesRead).first);
numTuplesRead++;
}
assertEquals(basicInputData.size(),numTuplesRead);
}
APIUtilityVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier Kfields=dumpedKSchema.getFields();
assertEquals(6,Kfields.size());
assertEquals(DataType.CHARARRAY,Kfields.get(0).type);
assertEquals("name",Kfields.get(0).alias.toLowerCase());
assertEquals(DataType.INTEGER,Kfields.get(1).type);
assertEquals("studentid",Kfields.get(1).alias.toLowerCase());
assertEquals(DataType.TUPLE,Kfields.get(2).type);
assertEquals("contact",Kfields.get(2).alias.toLowerCase());
{
assertNotNull(Kfields.get(2).schema);
assertTrue(Kfields.get(2).schema.getFields().size() == 2);
assertTrue(Kfields.get(2).schema.getFields().get(0).type == DataType.CHARARRAY);
assertTrue(Kfields.get(2).schema.getFields().get(0).alias.equalsIgnoreCase("phno"));
assertTrue(Kfields.get(2).schema.getFields().get(1).type == DataType.CHARARRAY);
assertTrue(Kfields.get(2).schema.getFields().get(1).alias.equalsIgnoreCase("email"));
}
assertEquals(DataType.BAG,Kfields.get(3).type);
assertEquals("currently_registered_courses",Kfields.get(3).alias.toLowerCase());
{
assertNotNull(Kfields.get(3).schema);
assertEquals(1,Kfields.get(3).schema.getFields().size());
assertEquals(DataType.TUPLE,Kfields.get(3).schema.getFields().get(0).type);
assertNotNull(Kfields.get(3).schema.getFields().get(0).schema);
assertEquals(1,Kfields.get(3).schema.getFields().get(0).schema.getFields().size());
assertEquals(DataType.CHARARRAY,Kfields.get(3).schema.getFields().get(0).schema.getFields().get(0).type);
}
assertEquals(DataType.MAP,Kfields.get(4).type);
assertEquals("current_grades",Kfields.get(4).alias.toLowerCase());
assertEquals(DataType.BAG,Kfields.get(5).type);
assertEquals("phnos",Kfields.get(5).alias.toLowerCase());
{
assertNotNull(Kfields.get(5).schema);
assertEquals(1,Kfields.get(5).schema.getFields().size());
assertEquals(DataType.TUPLE,Kfields.get(5).schema.getFields().get(0).type);
assertNotNull(Kfields.get(5).schema.getFields().get(0).schema);
assertTrue(Kfields.get(5).schema.getFields().get(0).schema.getFields().size() == 2);
assertEquals(DataType.CHARARRAY,Kfields.get(5).schema.getFields().get(0).schema.getFields().get(0).type);
assertEquals("phno",Kfields.get(5).schema.getFields().get(0).schema.getFields().get(0).alias.toLowerCase());
assertEquals(DataType.CHARARRAY,Kfields.get(5).schema.getFields().get(0).schema.getFields().get(1).type);
assertEquals("type",Kfields.get(5).schema.getFields().get(0).schema.getFields().get(1).alias.toLowerCase());
}
}
BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier Xfields=dumpedXSchema.getFields();
assertEquals(2,Xfields.size());
assertTrue(Xfields.get(0).alias.equalsIgnoreCase("a"));
assertTrue(Xfields.get(0).type == DataType.INTEGER);
assertTrue(Xfields.get(1).alias.equalsIgnoreCase("b"));
assertTrue(Xfields.get(1).type == DataType.CHARARRAY);
}
APIUtilityVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier iter=server.openIterator("E");
Tuple t=iter.next();
assertEquals(42,t.get(0));
t=iter.next();
assertEquals(1337,t.get(0));
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier valuesReadFromHiveDriver=new ArrayList();
driver.getResults(valuesReadFromHiveDriver);
assertEquals(basicInputData.size(),valuesReadFromHiveDriver.size());
server.registerQuery("W = load '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
Schema dumpedWSchema=server.dumpSchema("W");
List Wfields=dumpedWSchema.getFields();
assertEquals(3,Wfields.size());
assertTrue(Wfields.get(0).alias.equalsIgnoreCase("a"));
assertTrue(Wfields.get(0).type == DataType.INTEGER);
assertTrue(Wfields.get(1).alias.equalsIgnoreCase("b"));
assertTrue(Wfields.get(1).type == DataType.CHARARRAY);
assertTrue(Wfields.get(2).alias.equalsIgnoreCase("bkt"));
assertTrue(Wfields.get(2).type == DataType.CHARARRAY);
Iterator WIter=server.openIterator("W");
Collection> valuesRead=new ArrayList>();
while (WIter.hasNext()) {
Tuple t=WIter.next();
assertTrue(t.size() == 3);
assertNotNull(t.get(0));
assertNotNull(t.get(1));
assertNotNull(t.get(2));
assertTrue(t.get(0).getClass() == Integer.class);
assertTrue(t.get(1).getClass() == String.class);
assertTrue(t.get(2).getClass() == String.class);
valuesRead.add(new Pair((Integer)t.get(0),(String)t.get(1)));
if ((Integer)t.get(0) < 2) {
assertEquals("0",t.get(2));
}
else {
assertEquals("1",t.get(2));
}
}
assertEquals(valuesReadFromHiveDriver.size(),valuesRead.size());
server.registerQuery("P1 = load '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
server.registerQuery("P1filter = filter P1 by bkt == '0';");
Iterator P1Iter=server.openIterator("P1filter");
int count1=0;
while (P1Iter.hasNext()) {
Tuple t=P1Iter.next();
assertEquals("0",t.get(2));
assertEquals(1,t.get(0));
count1++;
}
assertEquals(3,count1);
server.registerQuery("P2 = load '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
server.registerQuery("P2filter = filter P2 by bkt == '1';");
Iterator P2Iter=server.openIterator("P2filter");
int count2=0;
while (P2Iter.hasNext()) {
Tuple t=P2Iter.next();
assertEquals("1",t.get(2));
assertTrue(((Integer)t.get(0)) > 1);
count2++;
}
assertEquals(6,count2);
}
APIUtilityVerifier BranchVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier iterator=server.openIterator("data");
Tuple t=iterator.next();
assertEquals("llama",t.get(0));
assertEquals(1,t.get(1));
t=iterator.next();
assertEquals("alpaca",t.get(0));
assertEquals(0,t.get(1));
assertFalse(iterator.hasNext());
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier XIter=server.openIterator("X");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(2,t.size());
assertNotNull(t.get(0));
assertNotNull(t.get(1));
assertTrue(t.get(0).getClass() == Integer.class);
assertTrue(t.get(1).getClass() == String.class);
assertEquals(t.get(0),basicInputData.get(numTuplesRead).first);
assertEquals(t.get(1),basicInputData.get(numTuplesRead).second);
numTuplesRead++;
}
assertEquals(basicInputData.size(),numTuplesRead);
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.hcatalog.pig.TestHCatLoaderEncryption APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier XIter=server.openIterator("X");
int numTuplesRead=0;
while (XIter.hasNext()) {
Tuple t=XIter.next();
assertEquals(2,t.size());
assertNotNull(t.get(0));
assertNotNull(t.get(1));
assertTrue(t.get(0).getClass() == Integer.class);
assertTrue(t.get(1).getClass() == String.class);
assertEquals(t.get(0),basicInputData.get(numTuplesRead).first);
assertEquals(t.get(1),basicInputData.get(numTuplesRead).second);
numTuplesRead++;
}
assertEquals("failed with storage format: " + this.storageFormat,basicInputData.size(),numTuplesRead);
}
BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.hcatalog.pig.TestHCatLoaderStorer APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier it=server.openIterator("data");
Tuple t=it.next();
Assert.assertEquals(new Integer(Short.MIN_VALUE),t.get(0));
Assert.assertEquals(new Integer(Byte.MIN_VALUE),t.get(1));
t=it.next();
Assert.assertEquals(new Integer(Short.MAX_VALUE),t.get(0));
Assert.assertEquals(new Integer(Byte.MAX_VALUE),t.get(1));
Assert.assertFalse(it.hasNext());
Assert.assertEquals(0,driver.run("drop table if exists " + writeTblName).getResponseCode());
Assert.assertEquals(0,driver.run("create table " + writeTblName + " (my_small_int smallint, my_tiny_int tinyint) stored as rcfile").getResponseCode());
HcatTestUtils.createTestDataFile(writeDataFile.getAbsolutePath(),new String[]{String.format("%d\t%d",Short.MIN_VALUE,Byte.MIN_VALUE),String.format("%d\t%d",Short.MAX_VALUE,Byte.MAX_VALUE)});
smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\","/"),ExecJob.JOB_STATUS.COMPLETED);
HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv",new String[]{String.format("%d\t%d",Short.MIN_VALUE - 1,0)});
smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/shortTooSmall.tsv",ExecJob.JOB_STATUS.FAILED);
HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooBig.tsv",new String[]{String.format("%d\t%d",Short.MAX_VALUE + 1,0)});
smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/shortTooBig.tsv",ExecJob.JOB_STATUS.FAILED);
HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/byteTooSmall.tsv",new String[]{String.format("%d\t%d",0,Byte.MIN_VALUE - 1)});
smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/byteTooSmall.tsv",ExecJob.JOB_STATUS.FAILED);
HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/byteTooBig.tsv",new String[]{String.format("%d\t%d",0,Byte.MAX_VALUE + 1)});
smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/byteTooBig.tsv",ExecJob.JOB_STATUS.FAILED);
}
Class: org.apache.hive.hcatalog.pig.TestHCatStorer APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier itr=server.openIterator("B");
int i=0;
while (itr.hasNext()) {
Tuple t=itr.next();
assertEquals(2,t.size());
assertEquals(t.get(0),i);
assertEquals(t.get(1),"1");
i++;
}
assertFalse(itr.hasNext());
assertEquals(LOOP_SIZE,i);
}
APIUtilityVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier , arr_of_struct array, " + "arr_of_struct2 array>, arr_of_struct3 array>) stored as " + storageFormat;
int retCode=driver.run(createTable).getResponseCode();
if (retCode != 0) {
throw new IOException("Failed to create table.");
}
String[] inputData=new String[]{"zookeeper\t(2)\t{(pig)}\t{(pnuts,hdfs)}\t{(hadoop),(hcat)}","chubby\t(2)\t{(sawzall)}\t{(bigtable,gfs)}\t{(mapreduce),(hcat)}"};
HcatTestUtils.createTestDataFile(INPUT_FILE_NAME,inputData);
PigServer server=new PigServer(ExecType.LOCAL);
server.setBatchOn();
server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (b:chararray, a:tuple(a1:int), arr_of_struct:bag{mytup:tuple(s1:chararray)}, arr_of_struct2:bag{mytup:tuple(s1:chararray,s2:chararray)}, arr_of_struct3:bag{t3:tuple(s3:chararray)});");
server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','b:chararray, a:tuple(a1:int),"+ " arr_of_struct:bag{mytup:tuple(s1:chararray)}, arr_of_struct2:bag{mytup:tuple(s1:chararray,s2:chararray)}, arr_of_struct3:bag{t3:tuple(s3:chararray)}');");
server.executeBatch();
driver.run("select * from junit_unparted");
ArrayList res=new ArrayList();
driver.getResults(res);
driver.run("drop table junit_unparted");
Iterator itr=res.iterator();
assertEquals("zookeeper\t{\"a1\":2}\t[\"pig\"]\t[{\"s1\":\"pnuts\",\"s2\":\"hdfs\"}]\t[{\"s3\":\"hadoop\"},{\"s3\":\"hcat\"}]",itr.next());
assertEquals("chubby\t{\"a1\":2}\t[\"sawzall\"]\t[{\"s1\":\"bigtable\",\"s2\":\"gfs\"}]\t[{\"s3\":\"mapreduce\"},{\"s3\":\"hcat\"}]",itr.next());
assertFalse(itr.hasNext());
}
IterativeVerifier AssumptionSetter EqualityVerifier HybridVerifier itr=server.openIterator("B");
int numRowsRead=0;
while (itr.hasNext()) {
Tuple t=itr.next();
StringBuilder rowFromPig=new StringBuilder();
for (int i=0; i < t.size(); i++) {
rowFromPig.append(t.get(i)).append("\t");
}
rowFromPig.setLength(rowFromPig.length() - 1);
assertEquals("Comparing Pig to Raw data",rows[numRowsRead],rowFromPig.toString());
numRowsRead++;
}
assertEquals("Expected " + NUM_ROWS + " rows; got "+ numRowsRead+ " file="+ INPUT_FILE_NAME,NUM_ROWS,numRowsRead);
}
APIUtilityVerifier IterativeVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
driver.run("select * from junit_unparted2");
ArrayList res2=new ArrayList();
driver.getResults(res2);
res.addAll(res2);
driver.run("drop table junit_unparted");
driver.run("drop table junit_unparted2");
Iterator itr=res.iterator();
for (int i=0; i < LOOP_SIZE * LOOP_SIZE; i++) {
assertEquals(input[i],itr.next());
}
assertFalse(itr.hasNext());
}
APIUtilityVerifier IterativeVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
driver.run("drop table junit_unparted");
Iterator itr=res.iterator();
for (int i=1; i <= LOOP_SIZE; i++) {
String si=i + "";
for (int j=1; j <= LOOP_SIZE; j++) {
assertEquals(si + "\t" + j,itr.next());
}
}
assertFalse(itr.hasNext());
}
APIUtilityVerifier IterativeVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
driver.run("drop table junit_unparted");
Iterator itr=res.iterator();
for (int i=0; i < LOOP_SIZE * LOOP_SIZE; i++) {
assertEquals(input[i],itr.next());
}
assertFalse(itr.hasNext());
}
AssumptionSetter EqualityVerifier HybridVerifier results=new ArrayList();
driver.getResults(results);
assertEquals(4,results.size());
Collections.sort(results);
assertEquals(inputData[0],results.get(0));
assertEquals(inputData[1],results.get(1));
assertEquals(inputData[2],results.get(2));
assertEquals(inputData[3],results.get(3));
driver.run("drop table employee");
}
APIUtilityVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier AssumptionSetter EqualityVerifier HybridVerifier results=new ArrayList();
driver.getResults(results);
assertEquals(4,results.size());
Collections.sort(results);
assertEquals(inputData[0],results.get(0));
assertEquals(inputData[1],results.get(1));
assertEquals(inputData[2],results.get(2));
assertEquals(inputData[3],results.get(3));
driver.run("drop table employee");
}
APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
Iterator itr=res.iterator();
String next=itr.next();
assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL",next);
assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tNULL",itr.next());
assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tNULL",itr.next());
assertFalse(itr.hasNext());
server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";");
Iterator iter=server.openIterator("B");
int count=0;
int num5nulls=0;
while (iter.hasNext()) {
Tuple t=iter.next();
if (t.get(6) == null) {
num5nulls++;
}
else {
assertTrue(t.get(6) instanceof DataByteArray);
}
assertNull(t.get(7));
count++;
}
assertEquals(3,count);
assertEquals(1,num5nulls);
driver.run("drop table junit_unparted");
}
AssumptionSetter EqualityVerifier HybridVerifier results=new ArrayList();
driver.getResults(results);
assertEquals(4,results.size());
Collections.sort(results);
assertEquals(inputData[0],results.get(0));
assertEquals(inputData[1],results.get(1));
assertEquals(inputData[2],results.get(2));
assertEquals(inputData[3],results.get(3));
driver.run("drop table employee");
}
APIUtilityVerifier IterativeVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier itr=server.openIterator("B");
int i=0;
while (itr.hasNext()) {
Tuple t=itr.next();
assertEquals(2,t.size());
assertEquals(t.get(0),i);
assertEquals(t.get(1),"1");
i++;
}
assertFalse(itr.hasNext());
assertEquals(11,i);
}
AssumptionSetter EqualityVerifier HybridVerifier results=new ArrayList();
driver.getResults(results);
assertEquals(0,results.size());
driver.run("drop table employee");
}
APIUtilityVerifier IterativeVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
driver.run("drop table junit_unparted");
Iterator itr=res.iterator();
for (int i=0; i < LOOP_SIZE * LOOP_SIZE; i++) {
assertEquals(input[i],itr.next());
}
assertFalse(itr.hasNext());
}
BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
assertEquals(0,res.size());
assertTrue((new File(TEST_WAREHOUSE_DIR + "/ptn_fail")).exists());
assertFalse((new File(TEST_WAREHOUSE_DIR + "/ptn_fail/b=math")).exists());
}
Class: org.apache.hive.hcatalog.pig.TestHCatStorerMulti AssumptionSetter EqualityVerifier HybridVerifier partitionedTableValuesReadFromHiveDriver=new ArrayList();
driver.getResults(partitionedTableValuesReadFromHiveDriver);
assertEquals(basicInputData.size(),partitionedTableValuesReadFromHiveDriver.size());
}
AssumptionSetter EqualityVerifier HybridVerifier unpartitionedTableValuesReadFromHiveDriver=new ArrayList();
driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
assertEquals(basicInputData.size(),unpartitionedTableValuesReadFromHiveDriver.size());
}
AssumptionSetter EqualityVerifier HybridVerifier unpartitionedTableValuesReadFromHiveDriver=new ArrayList();
driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
driver.run("select * from " + PARTITIONED_TABLE);
ArrayList partitionedTableValuesReadFromHiveDriver=new ArrayList();
driver.getResults(partitionedTableValuesReadFromHiveDriver);
assertEquals(basicInputData.size(),unpartitionedTableValuesReadFromHiveDriver.size());
assertEquals(basicInputData.size(),partitionedTableValuesReadFromHiveDriver.size());
}
Class: org.apache.hive.hcatalog.pig.TestHCatStorerWrapper APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier res=new ArrayList();
driver.getResults(res);
driver.run("drop table junit_external");
Iterator itr=res.iterator();
for (int i=1; i <= LOOP_SIZE; i++) {
String si=i + "";
for (int j=1; j <= LOOP_SIZE; j++) {
Assert.assertEquals(si + "\t" + j+ "\t"+ part_val,itr.next());
}
}
Assert.assertFalse(itr.hasNext());
}
Class: org.apache.hive.hcatalog.pig.TestPigHCatUtil APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.streaming.TestStreaming InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier > offsetMap=new HashMap>();
recordOffsets(conf,dbLocation,offsetMap);
txnBatch.beginNextTransaction();
txnBatch.write("name01,11,-Hello streaming".getBytes());
txnBatch.write("name21,21,-Welcome to streaming".getBytes());
txnBatch.write("name41,21,-more Streaming unlimited".getBytes());
txnBatch.write("name51,21,-even more Streaming unlimited".getBytes());
txnBatch.write("name02,12,--Hello streaming".getBytes());
txnBatch.write("name22,22,--Welcome to streaming".getBytes());
txnBatch.write("name42,22,--more Streaming unlimited".getBytes());
txnBatch.write("name52,22,--even more Streaming unlimited".getBytes());
txnBatch.write("name7,4,aWelcome to streaming".getBytes());
txnBatch.write("name8,5,amore Streaming unlimited".getBytes());
txnBatch.write("name9,6,aeven more Streaming unlimited".getBytes());
txnBatch.write("name10,7,bHello streaming".getBytes());
txnBatch.write("name11,8,bWelcome to streaming".getBytes());
txnBatch.write("name12,9,bmore Streaming unlimited".getBytes());
txnBatch.write("name13,10,beven more Streaming unlimited".getBytes());
txnBatch.commit();
recordOffsets(conf,dbLocation,offsetMap);
Path path=new Path(dbLocation);
Collection files=FileDump.getAllFilesInPath(path,conf);
for ( String file : files) {
if (file.contains("bucket_00000")) {
corruptSideFile(file,conf,offsetMap,"bucket_00000",-1);
}
else if (file.contains("bucket_00001")) {
corruptSideFile(file,conf,offsetMap,"bucket_00001",0);
}
else if (file.contains("bucket_00002")) {
corruptSideFile(file,conf,offsetMap,"bucket_00002",3);
}
else if (file.contains("bucket_00003")) {
corruptSideFile(file,conf,offsetMap,"bucket_00003",10);
}
}
PrintStream origErr=System.err;
ByteArrayOutputStream myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation});
System.err.flush();
System.setErr(origErr);
String errDump=new String(myErr.toByteArray());
Assert.assertEquals(true,errDump.contains("bucket_00000_flush_length [length: 11"));
Assert.assertEquals(true,errDump.contains("bucket_00001_flush_length [length: 0"));
Assert.assertEquals(true,errDump.contains("bucket_00002_flush_length [length: 24"));
Assert.assertEquals(true,errDump.contains("bucket_00003_flush_length [length: 80"));
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(true,errDump.contains("4 file(s) are corrupted"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
origErr=System.err;
myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation,"--recover","--skip-dump"});
System.err.flush();
System.setErr(origErr);
errDump=new String(myErr.toByteArray());
Assert.assertEquals(true,errDump.contains("bucket_00000 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("bucket_00001 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("bucket_00002 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("bucket_00003 recovered successfully!"));
List offsets=offsetMap.get("bucket_00000");
Assert.assertEquals(true,errDump.contains("Readable footerOffsets: " + offsets.toString()));
offsets=offsetMap.get("bucket_00001");
Assert.assertEquals(true,errDump.contains("Readable footerOffsets: " + offsets.toString()));
offsets=offsetMap.get("bucket_00002");
Assert.assertEquals(true,errDump.contains("Readable footerOffsets: " + offsets.toString()));
offsets=offsetMap.get("bucket_00003");
Assert.assertEquals(true,errDump.contains("Readable footerOffsets: " + offsets.toString()));
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
origErr=System.err;
myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation});
System.err.flush();
System.setErr(origErr);
errDump=new String(myErr.toByteArray());
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(false,errDump.contains("file(s) are corrupted"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
files=FileDump.getAllFilesInPath(path,conf);
for ( String file : files) {
Assert.assertEquals(false,file.contains("_flush_length"));
}
txnBatch.close();
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier > actual1=dumpAllBuckets(dbLocation,tblName3);
HashMap> actual2=dumpAllBuckets(dbLocation2,tblName4);
System.err.println("\n Table 1");
System.err.println(actual1);
System.err.println("\n Table 2");
System.err.println(actual2);
Assert.assertEquals("number of buckets does not match expectation",actual1.values().size(),4);
Assert.assertEquals("records in bucket does not match expectation",actual1.get(0).size(),2);
Assert.assertEquals("records in bucket does not match expectation",actual1.get(1).size(),1);
Assert.assertEquals("records in bucket does not match expectation",actual1.get(2).size(),0);
Assert.assertEquals("records in bucket does not match expectation",actual1.get(3).size(),1);
}
InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier files=FileDump.getAllFilesInPath(path,conf);
int readableFooter=-1;
for ( String file : files) {
if (file.contains("bucket_00000")) {
corruptDataFile(file,conf,Integer.MIN_VALUE);
}
else if (file.contains("bucket_00001")) {
corruptDataFile(file,conf,-1);
}
else if (file.contains("bucket_00002")) {
Path bPath=new Path(file);
FileSystem fs=bPath.getFileSystem(conf);
FileStatus fileStatus=fs.getFileStatus(bPath);
readableFooter=(int)fileStatus.getLen();
corruptDataFile(file,conf,2);
}
else if (file.contains("bucket_00003")) {
corruptDataFile(file,conf,100);
}
}
PrintStream origErr=System.err;
ByteArrayOutputStream myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation});
System.err.flush();
System.setErr(origErr);
String errDump=new String(myErr.toByteArray());
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(true,errDump.contains("4 file(s) are corrupted"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
origErr=System.err;
myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation,"--recover","--skip-dump"});
System.err.flush();
System.setErr(origErr);
errDump=new String(myErr.toByteArray());
Assert.assertEquals(true,errDump.contains("bucket_00000 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("No readable footers found. Creating empty orc file."));
Assert.assertEquals(true,errDump.contains("bucket_00001 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("bucket_00002 recovered successfully!"));
Assert.assertEquals(true,errDump.contains("Readable footerOffsets: [" + readableFooter + "]"));
Assert.assertEquals(true,errDump.contains("bucket_00003 recovered successfully!"));
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
origErr=System.err;
myErr=new ByteArrayOutputStream();
System.setErr(new PrintStream(myErr));
FileDump.main(new String[]{dbLocation});
System.err.flush();
System.setErr(origErr);
errDump=new String(myErr.toByteArray());
Assert.assertEquals(false,errDump.contains("Exception"));
Assert.assertEquals(false,errDump.contains("file(s) are corrupted"));
Assert.assertEquals(false,errDump.contains("is still open for writes."));
files=FileDump.getAllFilesInPath(path,conf);
for ( String file : files) {
Assert.assertEquals(false,file.contains("_flush_length"));
}
txnBatch.close();
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier ti=r.getOpen_txns();
Assert.assertEquals("wrong status ti(0)",TxnState.ABORTED,ti.get(0).getState());
Assert.assertEquals("wrong status ti(1)",TxnState.ABORTED,ti.get(1).getState());
Exception expectedEx=null;
try {
txnBatch.beginNextTransaction();
}
catch ( IllegalStateException ex) {
expectedEx=ex;
}
Assert.assertTrue("beginNextTransaction() should have failed",expectedEx != null && expectedEx.getMessage().contains("has been closed()"));
expectedEx=null;
try {
txnBatch.write("name0,1,Hello streaming".getBytes());
}
catch ( IllegalStateException ex) {
expectedEx=ex;
}
Assert.assertTrue("write() should have failed",expectedEx != null && expectedEx.getMessage().contains("has been closed()"));
expectedEx=null;
try {
txnBatch.commit();
}
catch ( IllegalStateException ex) {
expectedEx=ex;
}
Assert.assertTrue("commit() should have failed",expectedEx != null && expectedEx.getMessage().contains("has been closed()"));
txnBatch=connection.fetchTransactionBatch(2,writer);
txnBatch.beginNextTransaction();
txnBatch.write("name2,2,Welcome to streaming".getBytes());
txnBatch.write("name4,2,more Streaming unlimited".getBytes());
txnBatch.write("name5,2,even more Streaming unlimited".getBytes());
txnBatch.commit();
expectedEx=null;
txnBatch.beginNextTransaction();
writer.enableErrors();
try {
txnBatch.write("name6,2,Doh!".getBytes());
}
catch ( StreamingIOFailure ex) {
expectedEx=ex;
}
Assert.assertTrue("Wrong exception: " + (expectedEx != null ? expectedEx.getMessage() : "?"),expectedEx != null && expectedEx.getMessage().contains("Simulated fault occurred"));
expectedEx=null;
try {
txnBatch.commit();
}
catch ( IllegalStateException ex) {
expectedEx=ex;
}
Assert.assertTrue("commit() should have failed",expectedEx != null && expectedEx.getMessage().contains("has been closed()"));
r=msClient.showTxns();
Assert.assertEquals("HWM didn't match",4,r.getTxn_high_water_mark());
ti=r.getOpen_txns();
Assert.assertEquals("wrong status ti(0)",TxnState.ABORTED,ti.get(0).getState());
Assert.assertEquals("wrong status ti(1)",TxnState.ABORTED,ti.get(1).getState());
Assert.assertEquals("wrong status ti(2)",TxnState.ABORTED,ti.get(2).getState());
writer.disableErrors();
txnBatch=connection.fetchTransactionBatch(2,writer);
txnBatch.beginNextTransaction();
txnBatch.write("name2,2,Welcome to streaming".getBytes());
writer.enableErrors();
expectedEx=null;
try {
txnBatch.commit();
}
catch ( StreamingIOFailure ex) {
expectedEx=ex;
}
Assert.assertTrue("Wrong exception: " + (expectedEx != null ? expectedEx.getMessage() : "?"),expectedEx != null && expectedEx.getMessage().contains("Simulated fault occurred"));
r=msClient.showTxns();
Assert.assertEquals("HWM didn't match",6,r.getTxn_high_water_mark());
ti=r.getOpen_txns();
Assert.assertEquals("wrong status ti(3)",TxnState.ABORTED,ti.get(3).getState());
Assert.assertEquals("wrong status ti(4)",TxnState.ABORTED,ti.get(4).getState());
txnBatch.abort();
}
APIUtilityVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.hcatalog.streaming.mutate.client.lock.TestLock InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=request.getComponent();
assertEquals(2,components.size());
LockComponent expected1=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"DB");
expected1.setTablename("SOURCE_1");
assertTrue(components.contains(expected1));
LockComponent expected2=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"DB");
expected2.setTablename("SOURCE_2");
assertTrue(components.contains(expected2));
}
InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier components=request.getComponent();
assertEquals(3,components.size());
LockComponent expected1=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"DB");
expected1.setTablename("SOURCE_1");
assertTrue(components.contains(expected1));
LockComponent expected2=new LockComponent(LockType.SHARED_READ,LockLevel.TABLE,"DB");
expected2.setTablename("SOURCE_2");
assertTrue(components.contains(expected2));
LockComponent expected3=new LockComponent(LockType.SHARED_WRITE,LockLevel.TABLE,"DB");
expected3.setTablename("SINK");
assertTrue(components.contains(expected3));
}
Class: org.apache.hive.hcatalog.templeton.TestWebHCatE2e EqualityVerifier PublicFieldVerifier props=JsonBuilder.jsonToMap(p.responseBody);
Assert.assertEquals(p.getAssertMsg(),"Pig version request not yet " + "implemented",(String)props.get("error"));
}
APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier props=JsonBuilder.jsonToMap(p.responseBody);
Assert.assertEquals("hive",props.get("module"));
Assert.assertTrue(p.getAssertMsg(),((String)props.get("version")).matches("[0-9]+.[0-9]+.[0-9]+.*"));
}
APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier props=new HashMap();
props.put("comment","Table in default db");
Map col=new HashMap();
col.put("name","col1");
col.put("type","string");
List> colList=new ArrayList>(1);
colList.add(col);
props.put("columns",colList);
Map format=new HashMap();
format.put("storedAs","rcfile");
props.put("format",format);
MethodCallRetVal createTbl=doHttpCall(templetonBaseUrl + "/ddl/database/default/table/test_table",HTTP_METHOD_TYPE.PUT,props,null);
Assert.assertEquals(createTbl.getAssertMsg(),HttpStatus.OK_200,createTbl.httpStatusCode);
LOG.info("createTable() resp: " + createTbl.responseBody);
MethodCallRetVal descTbl=doHttpCall(templetonBaseUrl + "/ddl/database/default/table/test_table",HTTP_METHOD_TYPE.GET);
Assert.assertEquals(descTbl.getAssertMsg(),HttpStatus.OK_200,descTbl.httpStatusCode);
}
APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier props=new HashMap();
props.put("comment","Hello, there");
props.put("location",System.getProperty("test.warehouse.dir"));
Map props2=new HashMap();
props2.put("prop","val");
props.put("properties",props2);
MethodCallRetVal p=doHttpCall(templetonBaseUrl + "/ddl/database/newdb",HTTP_METHOD_TYPE.PUT,props,null);
Assert.assertEquals(p.getAssertMsg(),HttpStatus.OK_200,p.httpStatusCode);
}
APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier EqualityVerifier IgnoredMethod HybridVerifier EqualityVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier props=JsonBuilder.jsonToMap(p.responseBody);
Assert.assertEquals("hadoop",props.get("module"));
Assert.assertTrue(p.getAssertMsg(),((String)props.get("version")).matches("[1-2].[0-9]+.[0-9]+.*"));
}
APIUtilityVerifier EqualityVerifier IgnoredMethod HybridVerifier Class: org.apache.hive.hcatalog.templeton.tool.TestJobIDParser InternalCallVerifier EqualityVerifier jobs=hiveJobIDParser.parseJobID();
Assert.assertEquals(jobs.size(),1);
}
InternalCallVerifier EqualityVerifier jobs=pigJobIDParser.parseJobID();
Assert.assertEquals(jobs.size(),1);
}
InternalCallVerifier EqualityVerifier jobs=jarJobIDParser.parseJobID();
Assert.assertEquals(jobs.size(),1);
}
InternalCallVerifier EqualityVerifier jobs=jarJobIDParser.parseJobID();
Assert.assertEquals(jobs.size(),1);
}
Class: org.apache.hive.hcatalog.templeton.tool.TestTempletonUtils BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier EqualityVerifier Class: org.apache.hive.hcatalog.templeton.tool.TestTrivialExecService APIUtilityVerifier IterativeVerifier UtilityVerifier EqualityVerifier HybridVerifier list=new ArrayList();
list.add("echo");
list.add("success");
BufferedReader out=null;
BufferedReader err=null;
try {
Process process=TrivialExecService.getInstance().run(list,new ArrayList(),new HashMap());
out=new BufferedReader(new InputStreamReader(process.getInputStream()));
err=new BufferedReader(new InputStreamReader(process.getErrorStream()));
Assert.assertEquals("success",out.readLine());
out.close();
String line;
while ((line=err.readLine()) != null) {
Assert.fail(line);
}
process.waitFor();
}
catch ( Exception e) {
e.printStackTrace();
Assert.fail("Process caused exception.");
}
finally {
try {
out.close();
}
catch ( Exception ex) {
}
try {
err.close();
}
catch ( Exception ex) {
}
}
}
Class: org.apache.hive.jdbc.HiveStatementTest InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.jdbc.TestJdbcDriver2 APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BranchVerifier EqualityVerifier tests=new HashMap();
tests.put(new String[]{"testhivejdbcdriver\\_table",null},2);
tests.put(new String[]{"testhivejdbc%",null},8);
tests.put(new String[]{"testhiveJDBC%",null},8);
tests.put(new String[]{"%jdbcdriver\\_table",null},2);
tests.put(new String[]{"%jdbcdriver\\_table%","under\\_col"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","under\\_co_"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","under_col"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","und%"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","%"},2);
tests.put(new String[]{"%jdbcdriver\\_table%","_%"},2);
for ( String[] checkPattern : tests.keySet()) {
ResultSet rs=con.getMetaData().getColumns(null,"default",checkPattern[0],checkPattern[1]);
ResultSetMetaData rsmd=rs.getMetaData();
assertEquals("TABLE_CAT",rsmd.getColumnName(1));
int cnt=0;
while (rs.next()) {
String columnname=rs.getString("COLUMN_NAME");
int ordinalPos=rs.getInt("ORDINAL_POSITION");
switch (cnt) {
case 0:
assertEquals("Wrong column name found","under_col",columnname);
assertEquals("Wrong ordinal position found",ordinalPos,1);
break;
case 1:
assertEquals("Wrong column name found","value",columnname);
assertEquals("Wrong ordinal position found",ordinalPos,2);
break;
default :
break;
}
cnt++;
}
rs.close();
assertEquals("Found less columns then we test for.",tests.get(checkPattern).intValue(),cnt);
}
}
APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier ",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c6",meta.getColumnName(6));
assertEquals(Types.JAVA_OBJECT,meta.getColumnType(6));
assertEquals("map",meta.getColumnTypeName(6));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(6));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(6));
assertEquals(0,meta.getScale(6));
assertEquals("c6",colRS.getString("COLUMN_NAME"));
assertEquals(Types.JAVA_OBJECT,colRS.getInt("DATA_TYPE"));
assertEquals("map",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c7",meta.getColumnName(7));
assertEquals(Types.JAVA_OBJECT,meta.getColumnType(7));
assertEquals("map",meta.getColumnTypeName(7));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(7));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(7));
assertEquals(0,meta.getScale(7));
assertEquals("c7",colRS.getString("COLUMN_NAME"));
assertEquals(Types.JAVA_OBJECT,colRS.getInt("DATA_TYPE"));
assertEquals("map",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c8",meta.getColumnName(8));
assertEquals(Types.STRUCT,meta.getColumnType(8));
assertEquals("struct",meta.getColumnTypeName(8));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(8));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(8));
assertEquals(0,meta.getScale(8));
assertEquals("c8",colRS.getString("COLUMN_NAME"));
assertEquals(Types.STRUCT,colRS.getInt("DATA_TYPE"));
assertEquals("struct",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c9",meta.getColumnName(9));
assertEquals(Types.TINYINT,meta.getColumnType(9));
assertEquals("tinyint",meta.getColumnTypeName(9));
assertEquals(4,meta.getColumnDisplaySize(9));
assertEquals(3,meta.getPrecision(9));
assertEquals(0,meta.getScale(9));
assertEquals("c9",colRS.getString("COLUMN_NAME"));
assertEquals(Types.TINYINT,colRS.getInt("DATA_TYPE"));
assertEquals("tinyint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(9),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(9),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c10",meta.getColumnName(10));
assertEquals(Types.SMALLINT,meta.getColumnType(10));
assertEquals("smallint",meta.getColumnTypeName(10));
assertEquals(6,meta.getColumnDisplaySize(10));
assertEquals(5,meta.getPrecision(10));
assertEquals(0,meta.getScale(10));
assertEquals("c10",colRS.getString("COLUMN_NAME"));
assertEquals(Types.SMALLINT,colRS.getInt("DATA_TYPE"));
assertEquals("smallint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(10),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(10),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c11",meta.getColumnName(11));
assertEquals(Types.FLOAT,meta.getColumnType(11));
assertEquals("float",meta.getColumnTypeName(11));
assertEquals(24,meta.getColumnDisplaySize(11));
assertEquals(7,meta.getPrecision(11));
assertEquals(7,meta.getScale(11));
assertEquals("c11",colRS.getString("COLUMN_NAME"));
assertEquals(Types.FLOAT,colRS.getInt("DATA_TYPE"));
assertEquals("float",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(11),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(11),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c12",meta.getColumnName(12));
assertEquals(Types.BIGINT,meta.getColumnType(12));
assertEquals("bigint",meta.getColumnTypeName(12));
assertEquals(20,meta.getColumnDisplaySize(12));
assertEquals(19,meta.getPrecision(12));
assertEquals(0,meta.getScale(12));
assertEquals("c12",colRS.getString("COLUMN_NAME"));
assertEquals(Types.BIGINT,colRS.getInt("DATA_TYPE"));
assertEquals("bigint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(12),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(12),colRS.getInt("DECIMAL_DIGITS"));
assertEquals("c12_1",meta.getColumnName(13));
assertEquals(Types.INTEGER,meta.getColumnType(13));
assertEquals("int",meta.getColumnTypeName(13));
assertEquals(11,meta.getColumnDisplaySize(13));
assertEquals(10,meta.getPrecision(13));
assertEquals(0,meta.getScale(13));
assertEquals("b",meta.getColumnName(14));
assertEquals(Types.ARRAY,meta.getColumnType(14));
assertEquals("array",meta.getColumnTypeName(14));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(14));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(14));
assertEquals(0,meta.getScale(14));
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertEquals("c17",meta.getColumnName(15));
assertEquals(Types.TIMESTAMP,meta.getColumnType(15));
assertEquals("timestamp",meta.getColumnTypeName(15));
assertEquals(29,meta.getColumnDisplaySize(15));
assertEquals(29,meta.getPrecision(15));
assertEquals(9,meta.getScale(15));
assertEquals("c17",colRS.getString("COLUMN_NAME"));
assertEquals(Types.TIMESTAMP,colRS.getInt("DATA_TYPE"));
assertEquals("timestamp",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(15),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(15),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c18",meta.getColumnName(16));
assertEquals(Types.DECIMAL,meta.getColumnType(16));
assertEquals("decimal",meta.getColumnTypeName(16));
assertEquals(18,meta.getColumnDisplaySize(16));
assertEquals(16,meta.getPrecision(16));
assertEquals(7,meta.getScale(16));
assertEquals("c18",colRS.getString("COLUMN_NAME"));
assertEquals(Types.DECIMAL,colRS.getInt("DATA_TYPE"));
assertEquals("decimal",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(16),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(16),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertTrue(colRS.next());
assertEquals("c20",meta.getColumnName(17));
assertEquals(Types.DATE,meta.getColumnType(17));
assertEquals("date",meta.getColumnTypeName(17));
assertEquals(10,meta.getColumnDisplaySize(17));
assertEquals(10,meta.getPrecision(17));
assertEquals(0,meta.getScale(17));
assertEquals("c20",colRS.getString("COLUMN_NAME"));
assertEquals(Types.DATE,colRS.getInt("DATA_TYPE"));
assertEquals("date",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(17),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(17),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c21",meta.getColumnName(18));
assertEquals(Types.VARCHAR,meta.getColumnType(18));
assertEquals("varchar",meta.getColumnTypeName(18));
assertEquals(20,meta.getColumnDisplaySize(18));
assertEquals(20,meta.getPrecision(18));
assertEquals(0,meta.getScale(18));
assertEquals("c21",colRS.getString("COLUMN_NAME"));
assertEquals(Types.VARCHAR,colRS.getInt("DATA_TYPE"));
assertEquals("varchar",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(18),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(18),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c22",meta.getColumnName(19));
assertEquals(Types.CHAR,meta.getColumnType(19));
assertEquals("char",meta.getColumnTypeName(19));
assertEquals(15,meta.getColumnDisplaySize(19));
assertEquals(15,meta.getPrecision(19));
assertEquals(0,meta.getScale(19));
assertEquals("c23",meta.getColumnName(20));
assertEquals(Types.BINARY,meta.getColumnType(20));
assertEquals("binary",meta.getColumnTypeName(20));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(20));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(20));
assertEquals(0,meta.getScale(20));
assertEquals("c22",colRS.getString("COLUMN_NAME"));
assertEquals(Types.CHAR,colRS.getInt("DATA_TYPE"));
assertEquals("char",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(19),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(19),colRS.getInt("DECIMAL_DIGITS"));
for (int i=1; i <= meta.getColumnCount(); i++) {
assertFalse(meta.isAutoIncrement(i));
assertFalse(meta.isCurrency(i));
assertEquals(ResultSetMetaData.columnNullable,meta.isNullable(i));
}
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.TestJdbcWithLocalClusterSpark APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.TestJdbcWithMiniHS2 BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.TestJdbcWithMiniMr APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.TestSSL UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier BooleanVerifier InternalCallVerifier AssumptionSetter EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.jdbc.TestServiceDiscovery APIUtilityVerifier EqualityVerifier confs=new HashMap();
confs.put("hive.server2.thrift.bind.host","host-1");
confs.put("hive.server2.transport.mode","binary");
confs.put("hive.server2.thrift.port","8000");
confs.put("hive.server2.authentication","PLAIN");
publishConfsToZk(confs,"uri1");
confs.clear();
confs.put("hive.server2.thrift.bind.host","host-2");
confs.put("hive.server2.transport.mode","binary");
confs.put("hive.server2.thrift.port","9000");
confs.put("hive.server2.authentication","PLAIN");
publishConfsToZk(confs,"uri2");
confs.clear();
confs.put("hive.server2.thrift.bind.host","host-3");
confs.put("hive.server2.transport.mode","binary");
confs.put("hive.server2.thrift.port","10000");
confs.put("hive.server2.authentication","PLAIN");
publishConfsToZk(confs,"uri3");
Utils.JdbcConnectionParams connParams=new Utils.JdbcConnectionParams();
connParams.setZooKeeperEnsemble(server.getConnectString());
connParams.getSessionVars().put(Utils.JdbcConnectionParams.ZOOKEEPER_NAMESPACE,"hiveserver2");
List allConnectParams=new ArrayList<>();
while (true) {
try {
ZooKeeperHiveClientHelper.configureConnParams(connParams);
}
catch ( ZooKeeperHiveClientException e) {
break;
}
connParams.getRejectedHostZnodePaths().add(connParams.getCurrentHostZnodePath());
allConnectParams.add(new ConnParamInfo(connParams.getHost(),connParams.getPort(),connParams.getCurrentHostZnodePath()));
}
Collection cp1=Collections2.filter(allConnectParams,new ConnParamInfoPred("host-1",8000,"serverUri=uri1"));
Collection cp2=Collections2.filter(allConnectParams,new ConnParamInfoPred("host-2",9000,"serverUri=uri2"));
Collection cp3=Collections2.filter(allConnectParams,new ConnParamInfoPred("host-3",10000,"serverUri=uri3"));
Assert.assertEquals(cp1.size(),1);
Assert.assertEquals(cp2.size(),1);
Assert.assertEquals(cp3.size(),1);
}
Class: org.apache.hive.jdbc.authorization.TestCLIAuthzSessionContext BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.authorization.TestHS2AuthzSessionContext BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.jdbc.cbo_rp_TestJdbcDriver2 APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier ",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c6",meta.getColumnName(6));
assertEquals(Types.JAVA_OBJECT,meta.getColumnType(6));
assertEquals("map",meta.getColumnTypeName(6));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(6));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(6));
assertEquals(0,meta.getScale(6));
assertEquals("c6",colRS.getString("COLUMN_NAME"));
assertEquals(Types.JAVA_OBJECT,colRS.getInt("DATA_TYPE"));
assertEquals("map",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c7",meta.getColumnName(7));
assertEquals(Types.JAVA_OBJECT,meta.getColumnType(7));
assertEquals("map",meta.getColumnTypeName(7));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(7));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(7));
assertEquals(0,meta.getScale(7));
assertEquals("c7",colRS.getString("COLUMN_NAME"));
assertEquals(Types.JAVA_OBJECT,colRS.getInt("DATA_TYPE"));
assertEquals("map",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c8",meta.getColumnName(8));
assertEquals(Types.STRUCT,meta.getColumnType(8));
assertEquals("struct",meta.getColumnTypeName(8));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(8));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(8));
assertEquals(0,meta.getScale(8));
assertEquals("c8",colRS.getString("COLUMN_NAME"));
assertEquals(Types.STRUCT,colRS.getInt("DATA_TYPE"));
assertEquals("struct",colRS.getString("TYPE_NAME").toLowerCase());
assertTrue(colRS.next());
assertEquals("c9",meta.getColumnName(9));
assertEquals(Types.TINYINT,meta.getColumnType(9));
assertEquals("tinyint",meta.getColumnTypeName(9));
assertEquals(4,meta.getColumnDisplaySize(9));
assertEquals(3,meta.getPrecision(9));
assertEquals(0,meta.getScale(9));
assertEquals("c9",colRS.getString("COLUMN_NAME"));
assertEquals(Types.TINYINT,colRS.getInt("DATA_TYPE"));
assertEquals("tinyint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(9),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(9),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c10",meta.getColumnName(10));
assertEquals(Types.SMALLINT,meta.getColumnType(10));
assertEquals("smallint",meta.getColumnTypeName(10));
assertEquals(6,meta.getColumnDisplaySize(10));
assertEquals(5,meta.getPrecision(10));
assertEquals(0,meta.getScale(10));
assertEquals("c10",colRS.getString("COLUMN_NAME"));
assertEquals(Types.SMALLINT,colRS.getInt("DATA_TYPE"));
assertEquals("smallint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(10),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(10),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c11",meta.getColumnName(11));
assertEquals(Types.FLOAT,meta.getColumnType(11));
assertEquals("float",meta.getColumnTypeName(11));
assertEquals(24,meta.getColumnDisplaySize(11));
assertEquals(7,meta.getPrecision(11));
assertEquals(7,meta.getScale(11));
assertEquals("c11",colRS.getString("COLUMN_NAME"));
assertEquals(Types.FLOAT,colRS.getInt("DATA_TYPE"));
assertEquals("float",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(11),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(11),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c12",meta.getColumnName(12));
assertEquals(Types.BIGINT,meta.getColumnType(12));
assertEquals("bigint",meta.getColumnTypeName(12));
assertEquals(20,meta.getColumnDisplaySize(12));
assertEquals(19,meta.getPrecision(12));
assertEquals(0,meta.getScale(12));
assertEquals("c12",colRS.getString("COLUMN_NAME"));
assertEquals(Types.BIGINT,colRS.getInt("DATA_TYPE"));
assertEquals("bigint",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(12),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(12),colRS.getInt("DECIMAL_DIGITS"));
assertEquals("_c12",meta.getColumnName(13));
assertEquals(Types.INTEGER,meta.getColumnType(13));
assertEquals("int",meta.getColumnTypeName(13));
assertEquals(11,meta.getColumnDisplaySize(13));
assertEquals(10,meta.getPrecision(13));
assertEquals(0,meta.getScale(13));
assertEquals("b",meta.getColumnName(14));
assertEquals(Types.ARRAY,meta.getColumnType(14));
assertEquals("array",meta.getColumnTypeName(14));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(14));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(14));
assertEquals(0,meta.getScale(14));
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertTrue(colRS.next());
assertEquals("c17",meta.getColumnName(15));
assertEquals(Types.TIMESTAMP,meta.getColumnType(15));
assertEquals("timestamp",meta.getColumnTypeName(15));
assertEquals(29,meta.getColumnDisplaySize(15));
assertEquals(29,meta.getPrecision(15));
assertEquals(9,meta.getScale(15));
assertEquals("c17",colRS.getString("COLUMN_NAME"));
assertEquals(Types.TIMESTAMP,colRS.getInt("DATA_TYPE"));
assertEquals("timestamp",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(15),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(15),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c18",meta.getColumnName(16));
assertEquals(Types.DECIMAL,meta.getColumnType(16));
assertEquals("decimal",meta.getColumnTypeName(16));
assertEquals(18,meta.getColumnDisplaySize(16));
assertEquals(16,meta.getPrecision(16));
assertEquals(7,meta.getScale(16));
assertEquals("c18",colRS.getString("COLUMN_NAME"));
assertEquals(Types.DECIMAL,colRS.getInt("DATA_TYPE"));
assertEquals("decimal",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(16),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(16),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertTrue(colRS.next());
assertEquals("c20",meta.getColumnName(17));
assertEquals(Types.DATE,meta.getColumnType(17));
assertEquals("date",meta.getColumnTypeName(17));
assertEquals(10,meta.getColumnDisplaySize(17));
assertEquals(10,meta.getPrecision(17));
assertEquals(0,meta.getScale(17));
assertEquals("c20",colRS.getString("COLUMN_NAME"));
assertEquals(Types.DATE,colRS.getInt("DATA_TYPE"));
assertEquals("date",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(17),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(17),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c21",meta.getColumnName(18));
assertEquals(Types.VARCHAR,meta.getColumnType(18));
assertEquals("varchar",meta.getColumnTypeName(18));
assertEquals(20,meta.getColumnDisplaySize(18));
assertEquals(20,meta.getPrecision(18));
assertEquals(0,meta.getScale(18));
assertEquals("c21",colRS.getString("COLUMN_NAME"));
assertEquals(Types.VARCHAR,colRS.getInt("DATA_TYPE"));
assertEquals("varchar",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(18),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(18),colRS.getInt("DECIMAL_DIGITS"));
assertTrue(colRS.next());
assertEquals("c22",meta.getColumnName(19));
assertEquals(Types.CHAR,meta.getColumnType(19));
assertEquals("char",meta.getColumnTypeName(19));
assertEquals(15,meta.getColumnDisplaySize(19));
assertEquals(15,meta.getPrecision(19));
assertEquals(0,meta.getScale(19));
assertEquals("c23",meta.getColumnName(20));
assertEquals(Types.BINARY,meta.getColumnType(20));
assertEquals("binary",meta.getColumnTypeName(20));
assertEquals(Integer.MAX_VALUE,meta.getColumnDisplaySize(20));
assertEquals(Integer.MAX_VALUE,meta.getPrecision(20));
assertEquals(0,meta.getScale(20));
assertEquals("c22",colRS.getString("COLUMN_NAME"));
assertEquals(Types.CHAR,colRS.getInt("DATA_TYPE"));
assertEquals("char",colRS.getString("TYPE_NAME").toLowerCase());
assertEquals(meta.getPrecision(19),colRS.getInt("COLUMN_SIZE"));
assertEquals(meta.getScale(19),colRS.getInt("DECIMAL_DIGITS"));
for (int i=1; i <= meta.getColumnCount(); i++) {
assertFalse(meta.isAutoIncrement(i));
assertFalse(meta.isCurrency(i));
assertEquals(ResultSetMetaData.columnNullable,meta.isNullable(i));
}
}
APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BranchVerifier EqualityVerifier tests=new HashMap();
tests.put(new String[]{"testhivejdbcdriver\\_table",null},2);
tests.put(new String[]{"testhivejdbc%",null},7);
tests.put(new String[]{"testhiveJDBC%",null},7);
tests.put(new String[]{"%jdbcdriver\\_table",null},2);
tests.put(new String[]{"%jdbcdriver\\_table%","under\\_col"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","under\\_co_"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","under_col"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","und%"},1);
tests.put(new String[]{"%jdbcdriver\\_table%","%"},2);
tests.put(new String[]{"%jdbcdriver\\_table%","_%"},2);
for ( String[] checkPattern : tests.keySet()) {
ResultSet rs=con.getMetaData().getColumns(null,null,checkPattern[0],checkPattern[1]);
ResultSetMetaData rsmd=rs.getMetaData();
assertEquals("TABLE_CAT",rsmd.getColumnName(1));
int cnt=0;
while (rs.next()) {
String columnname=rs.getString("COLUMN_NAME");
int ordinalPos=rs.getInt("ORDINAL_POSITION");
switch (cnt) {
case 0:
assertEquals("Wrong column name found","under_col",columnname);
assertEquals("Wrong ordinal position found",ordinalPos,1);
break;
case 1:
assertEquals("Wrong column name found","value",columnname);
assertEquals("Wrong ordinal position found",ordinalPos,2);
break;
default :
break;
}
cnt++;
}
rs.close();
assertEquals("Found less columns then we test for.",tests.get(checkPattern).intValue(),cnt);
}
}
APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier NullVerifier PublicFieldVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier BooleanVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.jdbc.miniHS2.TestHiveServer2 InternalCallVerifier EqualityVerifier Class: org.apache.hive.minikdc.TestHiveAuthFactory InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.minikdc.TestHs2HooksWithMiniKdc BooleanVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.minikdc.TestJdbcWithMiniKdc UtilityVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.minikdc.TestMiniHiveKdc BooleanVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.ptest.execution.TestExecutionPhase EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier Class: org.apache.hive.ptest.execution.TestHostExecutor BooleanVerifier EqualityVerifier HybridVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.ptest.execution.TestJIRAService EqualityVerifier messages=Lists.newArrayList();
Assert.assertEquals(messages,JIRAService.trimMessages(messages));
messages.clear();
for (int i=0; i < JIRAService.MAX_MESSAGES; i++) {
messages.add(String.valueOf(i));
}
Assert.assertEquals(messages,JIRAService.trimMessages(messages));
}
EqualityVerifier messages=Lists.newArrayList("a","b","c");
Assert.assertEquals(messages,JIRAService.trimMessages(messages));
}
APIUtilityVerifier EqualityVerifier messages=Lists.newArrayList();
for (int i=0; i < JIRAService.MAX_MESSAGES + 1; i++) {
messages.add(String.valueOf(i));
}
List expected=Lists.newArrayList(messages);
expected.remove(0);
expected.add(0,JIRAService.TRIMMED_MESSAGE);
Assert.assertEquals(expected,JIRAService.trimMessages(messages));
}
InternalCallVerifier EqualityVerifier Class: org.apache.hive.ptest.execution.TestLocalCommand InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.ptest.execution.TestPhase EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier EqualityVerifier commands=localCommandFactory.getCommands();
Assert.assertEquals(1,commands.size());
Assert.assertEquals("local",commands.get(0));
}
Class: org.apache.hive.ptest.execution.TestReportParser InternalCallVerifier EqualityVerifier Class: org.apache.hive.ptest.execution.TestTestCheckPhase BooleanVerifier EqualityVerifier HybridVerifier addedTests=new HashSet();
phase=new TestCheckPhase(hostExecutors,localCommandFactory,templateDefaults,patchFile,logger,addedTests);
phase.execute();
Assert.assertEquals(addedTests.size(),3);
Assert.assertTrue(addedTests.contains("TestCodahaleMetrics.java"));
Assert.assertTrue(addedTests.contains("TestMetaStoreMetrics.java"));
Assert.assertTrue(addedTests.contains("TestLegacyMetrics.java"));
}
BooleanVerifier EqualityVerifier HybridVerifier addedTests=new HashSet();
phase=new TestCheckPhase(hostExecutors,localCommandFactory,templateDefaults,patchFile,logger,addedTests);
phase.execute();
Assert.assertEquals(addedTests.size(),1);
Assert.assertTrue(addedTests.contains("unionall_unbalancedppd.q"));
}
EqualityVerifier addedTests=new HashSet();
phase=new TestCheckPhase(hostExecutors,localCommandFactory,templateDefaults,patchFile,logger,addedTests);
phase.execute();
Assert.assertEquals(addedTests.size(),0);
}
EqualityVerifier addedTests=new HashSet();
phase=new TestCheckPhase(hostExecutors,localCommandFactory,templateDefaults,patchFile,logger,addedTests);
phase.execute();
Assert.assertEquals(addedTests.size(),0);
}
Class: org.apache.hive.ptest.execution.conf.TestQFileTestBatch BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.ptest.execution.conf.TestTestConfiguration APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier expectedHosts=Sets.newHashSet(new Host("localhost","hiveptest",new String[]{"/home/hiveptest"},2));
ExecutionContext executionContext=execConf.getExecutionContextProvider().createExecutionContext();
Assert.assertEquals(expectedHosts,executionContext.getHosts());
Assert.assertEquals("/tmp/hive-ptest-units/working/dir",execConf.getWorkingDirectory());
Assert.assertEquals("/etc/hiveptest/conf",execConf.getProfileDirectory());
Assert.assertEquals("/tmp/hive-ptest-units/working/dir/logs",execConf.getGlobalLogDirectory());
Assert.assertEquals("/home/brock/.ssh/id_rsa",executionContext.getPrivateKey());
Assert.assertEquals("git://github.com/apache/hive.git",conf.getRepository());
Assert.assertEquals("apache-github",conf.getRepositoryName());
Assert.assertEquals("trunk",conf.getBranch());
Assert.assertEquals("/tmp/hive-ptest-units/working/dir/working",executionContext.getLocalWorkingDirectory());
Assert.assertEquals("-Dtest.continue.on.failure=true -Dtest.silent=false",conf.getAntArgs());
Assert.assertEquals("hadoop-1,hadoop-2",conf.getAdditionalProfiles());
Assert.assertNotNull(conf.toString());
Assert.assertEquals("",conf.getPatch());
conf.setPatch("Patch");
Assert.assertEquals("Patch",conf.getPatch());
conf.setRepository("Repository");
Assert.assertEquals("Repository",conf.getRepository());
conf.setRepositoryName("RepositoryName");
Assert.assertEquals("RepositoryName",conf.getRepositoryName());
conf.setBranch("Branch");
Assert.assertEquals("Branch",conf.getBranch());
conf.setAntArgs("AntArgs");
Assert.assertEquals("AntArgs",conf.getAntArgs());
}
APIUtilityVerifier EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier testHosts=new HashSet();
testHosts.add(testHost);
TestConfiguration conf=TestConfiguration.fromInputStream(Resources.getResource("test-configuration.properties").openStream(),LOG);
ExecutionContext execContext=new ExecutionContext(null,testHosts,"test",null);
PTest.Builder mPTestBuilder=new PTest.Builder();
PTest ptest=mPTestBuilder.build(conf,execContext,"1234",baseDir.newFolder(),null,null,null,null);
Map templateDefaults=ptest.getTemplateDefaults();
Assert.assertEquals("git://github.com/apache/hive.git",templateDefaults.get("repository"));
Assert.assertEquals("apache-github",templateDefaults.get("repositoryName"));
Assert.assertEquals("trunk",templateDefaults.get("branch"));
Assert.assertEquals("-Dtest.continue.on.failure=true -Dtest.silent=false",templateDefaults.get("antArgs"));
Assert.assertEquals("hadoop-1,hadoop-2",templateDefaults.get("additionalProfiles"));
}
Class: org.apache.hive.ptest.execution.conf.TestTestParser InternalCallVerifier EqualityVerifier testBatches=testParser.parse().get();
Assert.assertEquals(4,testBatches.size());
}
InternalCallVerifier EqualityVerifier testBatches=testParser.parse().get();
Assert.assertEquals(4,testBatches.size());
}
InternalCallVerifier EqualityVerifier testBatches=testParser.parse().get();
Assert.assertEquals(4,testBatches.size());
}
Class: org.apache.hive.ptest.execution.ssh.TestRSyncCommandExecutor BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier (){
@Override public Integer answer( InvocationOnMock invocation) throws Throwable {
executor.shutdownNow();
return Constants.EXIT_CODE_UNKNOWN;
}
}
);
RSyncCommand command=new RSyncCommand(executor,"privateKey","user","host",1,"local","remote",RSyncCommand.Type.FROM_LOCAL);
executor.execute(command);
Assert.assertTrue(executor.isShutdown());
Assert.assertEquals(Constants.EXIT_CODE_UNKNOWN,command.getExitCode());
if (command.getException() != null) {
throw new Exception("Unexpected exception during execution",command.getException());
}
verify(localCommand,never()).kill();
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.ptest.execution.ssh.TestSSHCommandExecutor BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier (){
@Override public Integer answer( InvocationOnMock invocation) throws Throwable {
executor.shutdownNow();
return Constants.EXIT_CODE_UNKNOWN;
}
}
);
SSHCommand command=new SSHCommand(executor,"privateKey","user","host",1,"whoami",true);
executor.execute(command);
Assert.assertTrue(executor.isShutdown());
Assert.assertEquals(Constants.EXIT_CODE_UNKNOWN,command.getExitCode());
if (command.getException() != null) {
throw new Exception("Unexpected exception during execution",command.getException());
}
verify(localCommand,never()).kill();
}
Class: org.apache.hive.service.cli.CLIServiceTest APIUtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier confOverlay=new HashMap();
String tableName="TEST_EXEC_ASYNC";
String columnDefinitions="(ID STRING)";
String queryString;
SessionHandle sessionHandle=setupTestData(tableName,columnDefinitions,confOverlay);
assertNotNull(sessionHandle);
OperationState state=null;
OperationHandle opHandle;
OperationStatus opStatus=null;
queryString="SET " + HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname + " = false";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeOperation(opHandle);
long longPollingTimeout;
longPollingTimeout=HiveConf.getTimeVar(new HiveConf(),HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT,TimeUnit.MILLISECONDS);
queryString="SELECT NON_EXISTING_COLUMN FROM " + tableName;
try {
runAsyncAndWait(sessionHandle,queryString,confOverlay,OperationState.ERROR,longPollingTimeout);
}
catch ( HiveSQLException e) {
}
queryString="CREATE TABLE NON_EXISTING_TAB (ID STRING) location 'invalid://localhost:10000/a/b/c'";
opStatus=runAsyncAndWait(sessionHandle,queryString,confOverlay,OperationState.ERROR,longPollingTimeout);
assertEquals(opStatus.getOperationException().getSQLState(),"08S01");
assertEquals(opStatus.getOperationException().getErrorCode(),1);
queryString="SELECT ID+1 FROM " + tableName;
runAsyncAndWait(sessionHandle,queryString,confOverlay,OperationState.FINISHED,longPollingTimeout);
longPollingTimeout=0;
queryString="SELECT ID+1 FROM " + tableName;
runAsyncAndWait(sessionHandle,queryString,confOverlay,OperationState.FINISHED,longPollingTimeout);
longPollingTimeout=500;
queryString="SELECT ID+1 FROM " + tableName;
runAsyncAndWait(sessionHandle,queryString,confOverlay,OperationState.FINISHED,longPollingTimeout);
queryString="SELECT ID+1 FROM " + tableName;
opHandle=client.executeStatementAsync(sessionHandle,queryString,confOverlay);
System.out.println("Cancelling " + opHandle);
client.cancelOperation(opHandle);
state=client.getOperationStatus(opHandle).getState();
System.out.println(opHandle + " after cancelling, state= " + state);
assertEquals("Query should be cancelled",OperationState.CANCELED,state);
queryString="DROP TABLE " + tableName;
client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeSession(sessionHandle);
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier UtilityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier ());
assertNotNull(sessionHandle);
String tabName="TEST_CONF_EXEC";
String tabNameVar="tabNameVar";
String setLockMgr="SET " + HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname + " = false";
OperationHandle opHandle=client.executeStatement(sessionHandle,setLockMgr,null);
client.closeOperation(opHandle);
String dropTable="DROP TABLE IF EXISTS " + tabName;
opHandle=client.executeStatement(sessionHandle,dropTable,null);
client.closeOperation(opHandle);
Map confOverlay=new HashMap();
confOverlay.put(tabNameVar,tabName);
String createTab="CREATE TABLE ${hiveconf:" + tabNameVar + "} (id int)";
opHandle=client.executeStatement(sessionHandle,createTab,confOverlay);
assertNotNull(opHandle);
assertEquals("Query should be finished",OperationState.FINISHED,client.getOperationStatus(opHandle).getState());
client.closeOperation(opHandle);
String selectTab="SELECT * FROM " + tabName;
opHandle=client.executeStatement(sessionHandle,selectTab,null);
assertNotNull(opHandle);
assertEquals("Query should be finished",OperationState.FINISHED,client.getOperationStatus(opHandle).getState());
client.closeOperation(opHandle);
selectTab="SELECT * FROM ${hiveconf:" + tabNameVar + "}";
try {
opHandle=client.executeStatement(sessionHandle,selectTab,null);
fail("Query should fail");
}
catch ( HiveSQLException e) {
}
dropTable="DROP TABLE IF EXISTS " + tabName;
opHandle=client.executeStatement(sessionHandle,dropTable,null);
client.closeOperation(opHandle);
client.closeSession(sessionHandle);
}
InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier confOverlay=new HashMap();
SessionHandle sessionHandle=client.openSession("tom","password",new HashMap());
assertNotNull(sessionHandle);
OperationHandle opHandle;
String queryString="SET " + HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname + " = false";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeOperation(opHandle);
queryString="DROP TABLE IF EXISTS TEST_EXEC";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeOperation(opHandle);
queryString="CREATE TABLE TEST_EXEC(ID STRING)";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeOperation(opHandle);
queryString="SELECT ID+1 FROM TEST_EXEC";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
assertEquals("Query should be finished",OperationState.FINISHED,client.getOperationStatus(opHandle).getState());
client.closeOperation(opHandle);
queryString="DROP TABLE IF EXISTS TEST_EXEC";
opHandle=client.executeStatement(sessionHandle,queryString,confOverlay);
client.closeOperation(opHandle);
client.closeSession(sessionHandle);
}
Class: org.apache.hive.service.cli.TestColumn InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier integerTypesAndValues=new LinkedHashMap();
integerTypesAndValues.put(Type.TINYINT_TYPE,Arrays.asList(Byte.MIN_VALUE,Byte.MAX_VALUE));
integerTypesAndValues.put(Type.SMALLINT_TYPE,Arrays.asList(Short.MIN_VALUE,Short.MIN_VALUE));
integerTypesAndValues.put(Type.INT_TYPE,Arrays.asList(Integer.MIN_VALUE,Integer.MAX_VALUE));
integerTypesAndValues.put(Type.BIGINT_TYPE,Arrays.asList(Long.MIN_VALUE,Long.MAX_VALUE));
for ( Map.Entry entry : integerTypesAndValues.entrySet()) {
Type type=(Type)entry.getKey();
List values=(List)entry.getValue();
Column c=new Column(type);
for ( Object v : values) {
c.addValue(type,v);
}
assertEquals(type,c.getType());
assertEquals(values.size(),c.size());
for (int i=0; i < c.size(); i++) {
assertEquals(values.get(i),c.get(i));
}
}
}
InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.service.cli.TestHiveSQLException APIUtilityVerifier IdentityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier IdentityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IdentityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier details=HiveSQLException.toString(ex1);
String[] tokens=details.get(0).split(":");
tokens[0]="*DummyException";
details.set(0,StringUtils.join(tokens,":"));
Throwable ex=HiveSQLException.toCause(details);
Assert.assertEquals(RuntimeException.class,ex.getClass());
}
APIUtilityVerifier IdentityVerifier EqualityVerifier HybridVerifier details=HiveSQLException.toString(ex1);
String[] tokens=details.get(1).split(":");
tokens[2]=null;
tokens[3]="-1";
details.set(1,StringUtils.join(tokens,":"));
Throwable ex=HiveSQLException.toCause(details);
Assert.assertSame(RuntimeException.class,ex.getClass());
Assert.assertEquals("exception1",ex.getMessage());
Assert.assertSame(UnsupportedOperationException.class,ex.getCause().getClass());
Assert.assertEquals("exception2",ex.getCause().getMessage());
}
Class: org.apache.hive.service.cli.TestRetryingThriftCLIServiceClient UtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier confOverlay=new HashMap();
RetryingThriftCLIServiceClientTest.handlerInst.callCount=0;
RetryingThriftCLIServiceClientTest.handlerInst.connectCount=0;
SessionHandle session=cliServiceClient.openSession("anonymous","anonymous");
}
catch ( HiveSQLException exc) {
exc.printStackTrace();
assertTrue(exc.getCause() instanceof TException);
assertEquals(1,RetryingThriftCLIServiceClientTest.handlerInst.callCount);
assertEquals(3,RetryingThriftCLIServiceClientTest.handlerInst.connectCount);
}
}
Class: org.apache.hive.service.cli.operation.TestOperationLoggingAPIWithMr APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier Class: org.apache.hive.service.cli.operation.TestOperationLoggingLayout APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier iter=rowSetLog.iterator();
while (iter.hasNext()) {
String row=iter.next()[0].toString();
Assert.assertEquals(true,row.matches("^(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
}
String queryString="set hive.server2.logging.operation.level=verbose";
client.executeStatement(sessionHandle,queryString,null);
operationHandle=client.executeStatement(sessionHandle,sqlCntStar,null);
rowSetLog=client.fetchResults(operationHandle,FetchOrientation.FETCH_FIRST,10,FetchType.LOG);
iter=rowSetLog.iterator();
while (iter.hasNext()) {
String row=iter.next()[0].toString();
Assert.assertEquals(true,row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
}
}
Class: org.apache.hive.service.cli.session.TestPluggableHiveSessionImpl BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.service.cli.session.TestSessionHooks EqualityVerifier sessConf=new HashMap();
sessConf.put(HiveAuthFactory.HS2_PROXY_USER,proxyUser);
sessionUserName=proxyUser;
SessionHandle sessionHandle=client.openSession(connectingUser,"foobar",sessConf);
Assert.assertEquals(1,SessionHookTest.runCount.get());
client.closeSession(sessionHandle);
}
EqualityVerifier emptyMap());
Assert.assertEquals(1,SessionHookTest.runCount.get());
client.closeSession(sessionHandle);
}
Class: org.apache.hive.service.cli.thrift.ThriftCLIServiceTest InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier opConf=new HashMap();
SessionHandle sessHandle=client.openSession(USERNAME,PASSWORD,opConf);
assertNotNull("Session handle should not be null",sessHandle);
String queryString="SET hive.lock.manager=" + "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager";
client.executeStatement(sessHandle,queryString,opConf);
queryString="DROP TABLE IF EXISTS TEST_EXEC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
queryString="CREATE TABLE TEST_EXEC_THRIFT(ID STRING)";
client.executeStatement(sessHandle,queryString,opConf);
queryString="SELECT ID+1 FROM TEST_EXEC_THRIFT";
OperationHandle opHandle=client.executeStatement(sessHandle,queryString,opConf);
assertNotNull(opHandle);
OperationStatus opStatus=client.getOperationStatus(opHandle);
assertNotNull(opStatus);
OperationState state=opStatus.getState();
assertEquals("Query should be finished",OperationState.FINISHED,state);
queryString="DROP TABLE TEST_EXEC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
client.closeSession(sessHandle);
}
IterativeVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier opConf=new HashMap();
SessionHandle sessHandle=client.openSession(USERNAME,PASSWORD,opConf);
assertNotNull("Session handle should not be null",sessHandle);
OperationHandle opHandle;
OperationStatus opStatus;
OperationState state=null;
String queryString="SET hive.lock.manager=" + "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager";
client.executeStatement(sessHandle,queryString,opConf);
queryString="DROP TABLE IF EXISTS TEST_EXEC_ASYNC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
queryString="CREATE TABLE TEST_EXEC_ASYNC_THRIFT(ID STRING)";
client.executeStatement(sessHandle,queryString,opConf);
queryString="SELECT ID+1 FROM TEST_EXEC_ASYNC_THRIFT";
System.out.println("Will attempt to execute: " + queryString);
opHandle=client.executeStatementAsync(sessHandle,queryString,opConf);
assertNotNull(opHandle);
boolean isQueryRunning=true;
long pollTimeout=System.currentTimeMillis() + 100000;
while (isQueryRunning) {
if (System.currentTimeMillis() > pollTimeout) {
System.out.println("Polling timed out");
break;
}
opStatus=client.getOperationStatus(opHandle);
assertNotNull(opStatus);
state=opStatus.getState();
System.out.println("Current state: " + state);
if (state == OperationState.CANCELED || state == OperationState.CLOSED || state == OperationState.FINISHED || state == OperationState.ERROR) {
isQueryRunning=false;
}
Thread.sleep(1000);
}
assertEquals("Query should be finished",OperationState.FINISHED,state);
queryString="CREATE TABLE NON_EXISTING_TAB (ID STRING) location 'hdfs://localhost:10000/a/b/c'";
System.out.println("Will attempt to execute: " + queryString);
opHandle=client.executeStatementAsync(sessHandle,queryString,opConf);
assertNotNull(opHandle);
opStatus=client.getOperationStatus(opHandle);
assertNotNull(opStatus);
isQueryRunning=true;
pollTimeout=System.currentTimeMillis() + 100000;
while (isQueryRunning) {
if (System.currentTimeMillis() > pollTimeout) {
System.out.println("Polling timed out");
break;
}
state=opStatus.getState();
System.out.println("Current state: " + state);
if (state == OperationState.CANCELED || state == OperationState.CLOSED || state == OperationState.FINISHED || state == OperationState.ERROR) {
isQueryRunning=false;
}
Thread.sleep(1000);
opStatus=client.getOperationStatus(opHandle);
}
assertEquals("Operation should be in error state",OperationState.ERROR,state);
assertEquals(opStatus.getOperationException().getSQLState(),"08S01");
assertEquals(opStatus.getOperationException().getErrorCode(),1);
queryString="DROP TABLE TEST_EXEC_ASYNC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
client.closeSession(sessHandle);
}
Class: org.apache.hive.service.cli.thrift.ThriftCliServiceTestWithCookie InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier opConf=new HashMap();
SessionHandle sessHandle=client.openSession(USERNAME,PASSWORD,opConf);
assertNotNull("Session handle should not be null",sessHandle);
String queryString="SET hive.lock.manager=" + "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager";
client.executeStatement(sessHandle,queryString,opConf);
queryString="DROP TABLE IF EXISTS TEST_EXEC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
queryString="CREATE TABLE TEST_EXEC_THRIFT(ID STRING)";
client.executeStatement(sessHandle,queryString,opConf);
queryString="SELECT ID+1 FROM TEST_EXEC_THRIFT";
OperationHandle opHandle=client.executeStatement(sessHandle,queryString,opConf);
assertNotNull(opHandle);
OperationStatus opStatus=client.getOperationStatus(opHandle);
assertNotNull(opStatus);
OperationState state=opStatus.getState();
assertEquals("Query should be finished",OperationState.FINISHED,state);
queryString="DROP TABLE TEST_EXEC_THRIFT";
client.executeStatement(sessHandle,queryString,opConf);
client.closeSession(sessHandle);
}
Class: org.apache.hive.service.server.TestHS2HttpServer APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.service.server.TestServerOptionsProcessor EqualityVerifier Class: org.apache.hive.spark.client.TestMetricsCollection InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier Class: org.apache.hive.spark.client.TestSparkClient InternalCallVerifier EqualityVerifier NullVerifier HybridVerifier InternalCallVerifier EqualityVerifier result=client.run(new SyncRpc());
assertEquals("Hello",result.get(TIMEOUT,TimeUnit.SECONDS));
}
}
);
}
APIUtilityVerifier InternalCallVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier listener=newListener();
JobHandle future=client.submit(new AsyncSparkJob());
future.addListener(listener);
future.get(TIMEOUT,TimeUnit.SECONDS);
MetricsCollection metrics=future.getMetrics();
assertEquals(1,metrics.getJobIds().size());
assertTrue(metrics.getAllMetrics().executorRunTime >= 0L);
verify(listener).onSparkJobStarted(same(future),eq(metrics.getJobIds().iterator().next()));
JobHandle.Listener listener2=newListener();
JobHandle future2=client.submit(new AsyncSparkJob());
future2.addListener(listener2);
future2.get(TIMEOUT,TimeUnit.SECONDS);
MetricsCollection metrics2=future2.getMetrics();
assertEquals(1,metrics2.getJobIds().size());
assertFalse(Objects.equal(metrics.getJobIds(),metrics2.getJobIds()));
assertTrue(metrics2.getAllMetrics().executorRunTime >= 0L);
verify(listener2).onSparkJobStarted(same(future2),eq(metrics2.getJobIds().iterator().next()));
}
}
);
}
InternalCallVerifier EqualityVerifier handle=client.submit(new SparkJob());
assertEquals(Long.valueOf(5L),handle.get(TIMEOUT,TimeUnit.SECONDS));
}
}
);
}
InternalCallVerifier EqualityVerifier handle=client.submit(new SparkJob());
assertEquals(Long.valueOf(5L),handle.get(TIMEOUT,TimeUnit.SECONDS));
}
}
);
}
BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier listener=newListener();
JobHandle handle=client.submit(new SimpleJob());
handle.addListener(listener);
assertEquals("hello",handle.get(TIMEOUT,TimeUnit.SECONDS));
assertFalse(((JobHandleImpl)handle).changeState(JobHandle.State.SENT));
verify(listener).onJobQueued(handle);
verify(listener).onJobStarted(handle);
verify(listener).onJobSucceeded(same(handle),eq(handle.get()));
}
}
);
}
Class: org.apache.hive.spark.client.rpc.TestKryoMessageCodec EqualityVerifier BooleanVerifier EqualityVerifier HybridVerifier APIUtilityVerifier EqualityVerifier APIUtilityVerifier EqualityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier Class: org.apache.hive.spark.client.rpc.TestRpc APIUtilityVerifier BooleanVerifier InternalCallVerifier EqualityVerifier HybridVerifier call=client.call(outbound,TestMessage.class);
TestMessage reply=call.get(10,TimeUnit.SECONDS);
assertEquals(outbound.message,reply.message);
TestMessage another=new TestMessage("Hello again!");
Future anotherCall=client.call(another,TestMessage.class);
TestMessage anotherReply=anotherCall.get(10,TimeUnit.SECONDS);
assertEquals(another.message,anotherReply.message);
String errorMsg="This is an error.";
try {
client.call(new ErrorCall(errorMsg)).get(10,TimeUnit.SECONDS);
}
catch ( ExecutionException ee) {
assertTrue(ee.getCause() instanceof RpcException);
assertTrue(ee.getCause().getMessage().indexOf(errorMsg) >= 0);
}
TestMessage serverMsg=new TestMessage("Hello from the server!");
Future serverCall=serverRpc.call(serverMsg,TestMessage.class);
TestMessage serverReply=serverCall.get(10,TimeUnit.SECONDS);
assertEquals(serverMsg.message,serverReply.message);
}
InternalCallVerifier EqualityVerifier call=clientRpc.call(outbound,TestMessage.class);
LOG.debug("Transferring messages...");
transfer(serverRpc,clientRpc);
TestMessage reply=call.get(10,TimeUnit.SECONDS);
assertEquals(outbound.message,reply.message);
}
EqualityVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier eConf=ImmutableMap.builder().putAll(emptyConfig).put(RpcConfiguration.RPC_SASL_OPT_PREFIX + "qop",Rpc.SASL_AUTH_CONF).build();
RpcServer server=autoClose(new RpcServer(eConf));
Rpc[] rpcs=createRpcConnection(server,eConf);
Rpc client=rpcs[1];
TestMessage outbound=new TestMessage("Hello World!");
Future call=client.call(outbound,TestMessage.class);
TestMessage reply=call.get(10,TimeUnit.SECONDS);
assertEquals(outbound.message,reply.message);
}
Class: org.apache.orc.impl.TestBitFieldReader APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.orc.impl.TestDynamicArray InternalCallVerifier EqualityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier Class: org.apache.orc.impl.TestInStream APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier Class: org.apache.orc.impl.TestIntegerCompressionReader APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.orc.impl.TestMemoryManager InternalCallVerifier EqualityVerifier Class: org.apache.orc.impl.TestRunLengthByteReader APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.orc.impl.TestRunLengthIntegerReader APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier Class: org.apache.orc.impl.TestSerializationUtils UtilityVerifier EqualityVerifier HybridVerifier APIUtilityVerifier InternalCallVerifier EqualityVerifier InternalCallVerifier EqualityVerifier IterativeVerifier EqualityVerifier Class: org.apache.orc.impl.TestStringRedBlackTree EqualityVerifier InternalCallVerifier EqualityVerifier EqualityVerifier Class: org.apache.orc.impl.TestZlib EqualityVerifier